Skip to content

Commit

Permalink
[BUGFIX] index correction process (#2565) (#2566)
Browse files Browse the repository at this point in the history
* [BUGFIX] index correction process



* fix: disable chaos deploy



* Revert "fix: disable chaos deploy"

This reverts commit 48aa7a9.

* fix: add condition for k3d ingress



---------

Signed-off-by: kpango <kpango@vdaas.org>
Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com>
Co-authored-by: Yusuke Kato <kpango@vdaas.org>
Co-authored-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com>
  • Loading branch information
3 people authored Aug 6, 2024
1 parent 678e98d commit 2fc9958
Show file tree
Hide file tree
Showing 58 changed files with 4,352 additions and 2,966 deletions.
5 changes: 5 additions & 0 deletions .gitfiles
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
.github/helm/values/vald-mirror-target.yaml
.github/helm/values/values-agent-sidecar.yaml
.github/helm/values/values-chaos.yaml
.github/helm/values/values-correction.yaml
.github/helm/values/values-index-management-jobs.yaml
.github/helm/values/values-lb.yaml
.github/helm/values/values-max-dim.yaml
Expand Down Expand Up @@ -86,7 +87,9 @@
.github/workflows/dockers-agent-sidecar-image.yml
.github/workflows/dockers-benchmark-job-image.yml
.github/workflows/dockers-benchmark-operator-image.yaml
.github/workflows/dockers-binfmt-image.yaml
.github/workflows/dockers-buildbase-image.yml
.github/workflows/dockers-buildkit-image.yaml
.github/workflows/dockers-ci-container-image.yml
.github/workflows/dockers-dev-container-image.yml
.github/workflows/dockers-discoverer-k8s-image.yml
Expand Down Expand Up @@ -535,7 +538,9 @@ dockers/agent/core/ngt/Dockerfile
dockers/agent/core/ngt/README.md
dockers/agent/sidecar/Dockerfile
dockers/agent/sidecar/README.md
dockers/binfmt/Dockerfile
dockers/buildbase/Dockerfile
dockers/buildkit/Dockerfile
dockers/ci/base/Dockerfile
dockers/ci/base/Dockerfile
dockers/ci/base/Dockerfile
Expand Down
17 changes: 12 additions & 5 deletions .github/actions/setup-k3d/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -81,11 +81,18 @@ runs:
shell: bash
id: start_k3d
run: |
if [ "${K3D_INGRESS_PORT}" == "0" ] && [ -z "${K3D_OPTIONS}" ]; then
K3D_OPTIONS="--k3s-arg \"--disable=traefik@server:*\""
fi
if [ "${K3D_INGRESS_PORT}" != "0" ] && [ -n "${K3D_OPTIONS}" ]; then
K3D_OPTIONS="--port ${K3D_INGRESS_PORT}:80@loadbalancer ${K3D_OPTIONS}"
if [ "${K3D_INGRESS_PORT}" == "0" ]; then
if [ -z "${K3D_OPTIONS}" ]; then
K3D_OPTIONS="--k3s-arg \"--disable=traefik@server:*\""
else
K3D_OPTIONS="${K3D_OPTIONS} --k3s-arg \"--disable=traefik@server:*\""
fi
else
if [ -z "${K3D_OPTIONS}" ]; then
K3D_OPTIONS="--port ${K3D_INGRESS_PORT}:80@loadbalancer"
else
K3D_OPTIONS="${K3D_OPTIONS} --port ${K3D_INGRESS_PORT}:80@loadbalancer"
fi
fi
make K3D_CLUSTER_NAME="${{ inputs.name }}" \
K3D_NODES="${{ inputs.agents }}" \
Expand Down
73 changes: 73 additions & 0 deletions .github/helm/values/values-correction.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
#
# Copyright (C) 2019-2024 vdaas.org vald team <vald@vdaas.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

defaults:
logging:
level: info
networkPolicy:
enabled: true
gateway:
lb:
enabled: true
minReplicas: 1
hpa:
enabled: false
resources:
requests:
cpu: 100m
memory: 50Mi
gateway_config:
index_replica: 2
agent:
minReplicas: 10
maxReplicas: 10
podManagementPolicy: Parallel
hpa:
enabled: false
resources:
requests:
cpu: 100m
memory: 50Mi
ngt:
auto_index_duration_limit: 2m
auto_index_check_duration: 30s
auto_index_length: 500
dimension: 784
discoverer:
minReplicas: 1
hpa:
enabled: false
resources:
requests:
cpu: 100m
memory: 50Mi
manager:
index:
replicas: 1
resources:
requests:
cpu: 100m
memory: 30Mi
indexer:
auto_index_duration_limit: 2m
auto_index_check_duration: 30s
auto_index_length: 1000
corrector:
enabled: true
# suspend because you do not want corrector to start automatically in CI
# instead run it manually
suspend: true
schedule: "1 2 3 4 5"
48 changes: 48 additions & 0 deletions .github/workflows/dockers-binfmt-image.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
#
# Copyright (C) 2019-2024 vdaas.org vald team <vald@vdaas.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: "Build docker image: binfmt"
on:
schedule:
- cron: "0 * * * *"
push:
branches:
- "main"
- "release/v*.*"
- "!release/v*.*.*"
tags:
- "*.*.*"
- "v*.*.*"
- "*.*.*-*"
- "v*.*.*-*"
pull_request:
paths:
- ".github/actions/docker-build/action.yaml"
- ".github/workflows/_docker-image.yaml"
- ".github/workflows/dockers-binfmt-image.yml"
- "dockers/binfmt/Dockerfile"
pull_request_target:
paths:
- ".github/actions/docker-build/action.yaml"
- ".github/workflows/_docker-image.yaml"
- ".github/workflows/dockers-binfmt-image.yml"
- "dockers/binfmt/Dockerfile"
jobs:
build:
uses: ./.github/workflows/_docker-image.yaml
with:
target: binfmt
platforms: linux/amd64,linux/arm64
secrets: inherit
48 changes: 48 additions & 0 deletions .github/workflows/dockers-buildkit-image.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
#
# Copyright (C) 2019-2024 vdaas.org vald team <vald@vdaas.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: "Build docker image: buildkit"
on:
schedule:
- cron: "0 * * * *"
push:
branches:
- "main"
- "release/v*.*"
- "!release/v*.*.*"
tags:
- "*.*.*"
- "v*.*.*"
- "*.*.*-*"
- "v*.*.*-*"
pull_request:
paths:
- ".github/actions/docker-build/action.yaml"
- ".github/workflows/_docker-image.yaml"
- ".github/workflows/dockers-buildkit-image.yml"
- "dockers/buildkit/Dockerfile"
pull_request_target:
paths:
- ".github/actions/docker-build/action.yaml"
- ".github/workflows/_docker-image.yaml"
- ".github/workflows/dockers-buildkit-image.yml"
- "dockers/buildkit/Dockerfile"
jobs:
build:
uses: ./.github/workflows/_docker-image.yaml
with:
target: buildkit
platforms: linux/amd64,linux/arm64
secrets: inherit
30 changes: 20 additions & 10 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -17,32 +17,35 @@
SHELL = bash
ORG ?= vdaas
NAME = vald
GOPKG = github.com/$(ORG)/$(NAME)
REPO = $(ORG)/$(NAME)
GOPKG = github.com/$(REPO)
DATETIME = $(eval DATETIME := $(shell date -u +%Y/%m/%d_%H:%M:%S%z))$(DATETIME)
TAG ?= latest
CRORG ?= $(ORG)
GHCRORG = ghcr.io/$(ORG)/$(NAME)
GHCRORG = ghcr.io/$(REPO)
AGENT_IMAGE = $(NAME)-agent
AGENT_NGT_IMAGE = $(NAME)-agent-ngt
AGENT_FAISS_IMAGE = $(NAME)-agent-faiss
AGENT_SIDECAR_IMAGE = $(NAME)-agent-sidecar
AGENT_IMAGE = $(NAME)-agent
BENCHMARK_JOB_IMAGE = $(NAME)-benchmark-job
BENCHMARK_OPERATOR_IMAGE = $(NAME)-benchmark-operator
BINFMT_IMAGE = $(NAME)-binfmt
BUILDBASE_IMAGE = $(NAME)-buildbase
BUILDKIT_IMAGE = $(NAME)-buildkit
CI_CONTAINER_IMAGE = $(NAME)-ci-container
DEV_CONTAINER_IMAGE = $(NAME)-dev-container
DISCOVERER_IMAGE = $(NAME)-discoverer-k8s
FILTER_GATEWAY_IMAGE = $(NAME)-filter-gateway
MIRROR_GATEWAY_IMAGE = $(NAME)-mirror-gateway
HELM_OPERATOR_IMAGE = $(NAME)-helm-operator
LB_GATEWAY_IMAGE = $(NAME)-lb-gateway
LOADTEST_IMAGE = $(NAME)-loadtest
INDEX_CORRECTION_IMAGE = $(NAME)-index-correction
INDEX_CREATION_IMAGE = $(NAME)-index-creation
INDEX_SAVE_IMAGE = $(NAME)-index-save
INDEX_OPERATOR_IMAGE = $(NAME)-index-operator
READREPLICA_ROTATE_IMAGE = $(NAME)-readreplica-rotate
INDEX_SAVE_IMAGE = $(NAME)-index-save
LB_GATEWAY_IMAGE = $(NAME)-lb-gateway
LOADTEST_IMAGE = $(NAME)-loadtest
MANAGER_INDEX_IMAGE = $(NAME)-manager-index
BENCHMARK_JOB_IMAGE = $(NAME)-benchmark-job
BENCHMARK_OPERATOR_IMAGE = $(NAME)-benchmark-operator
MIRROR_GATEWAY_IMAGE = $(NAME)-mirror-gateway
READREPLICA_ROTATE_IMAGE = $(NAME)-readreplica-rotate
MAINTAINER = "$(ORG).org $(NAME) team <$(NAME)@$(ORG).org>"

VERSION ?= $(eval VERSION := $(shell cat versions/VALD_VERSION))$(VERSION)
Expand Down Expand Up @@ -351,6 +354,13 @@ E2E_UPDATE_COUNT ?= 10
E2E_UPSERT_COUNT ?= 10
E2E_WAIT_FOR_CREATE_INDEX_DURATION ?= 8m
E2E_WAIT_FOR_START_TIMEOUT ?= 10m
E2E_SEARCH_FROM ?= 0
E2E_SEARCH_BY_ID_FROM ?= 0
E2E_INSERT_FROM ?= 0
E2E_UPDATE_FROM ?= 0
E2E_UPSERT_FROM ?= 0
E2E_GET_OBJECT_FROM ?= 0
E2E_REMOVE_FROM ?= 0

TEST_RESULT_DIR ?= /tmp

Expand Down
4 changes: 2 additions & 2 deletions Makefile.d/dependencies.mk
Original file line number Diff line number Diff line change
Expand Up @@ -225,12 +225,12 @@ update/hdf5:
.PHONY: update/vald
## update vald it's self version
update/vald:
curl -fsSL https://api.github.com/repos/vdaas/vald/releases/latest | grep -Po '"tag_name": "\K.*?(?=")' > $(ROOTDIR)/versions/VALD_VERSION
curl -fsSL https://api.github.com/repos/$(REPO)/releases/latest | grep -Po '"tag_name": "\K.*?(?=")' > $(ROOTDIR)/versions/VALD_VERSION

.PHONY: update/valdcli
## update vald client library made by clojure self version
update/valdcli:
curl -fsSL https://api.github.com/repos/vdaas/vald-client-clj/releases/latest | grep -Po '"tag_name": "\K.*?(?=")' > $(ROOTDIR)/versions/VALDCLI_VERSION
curl -fsSL https://api.github.com/repos/$(REPO)-client-clj/releases/latest | grep -Po '"tag_name": "\K.*?(?=")' > $(ROOTDIR)/versions/VALDCLI_VERSION

.PHONY: update/template
## update PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE
Expand Down
45 changes: 40 additions & 5 deletions Makefile.d/docker.mk
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,29 @@
.PHONY: docker/build
## build all docker images
docker/build: \
docker/build/agent-ngt \
docker/build/agent \
docker/build/agent-faiss \
docker/build/agent-ngt \
docker/build/agent-sidecar \
docker/build/benchmark-job \
docker/build/benchmark-operator \
docker/build/binfmt \
docker/build/buildbase \
docker/build/buildkit \
docker/build/ci-container \
docker/build/dev-container \
docker/build/discoverer-k8s \
docker/build/gateway-lb \
docker/build/gateway-filter \
docker/build/gateway-lb \
docker/build/gateway-mirror \
docker/build/index-correction \
docker/build/index-creation \
docker/build/index-operator \
docker/build/index-save \
docker/build/loadtest \
docker/build/manager-index \
docker/build/benchmark-job \
docker/build/benchmark-operator \
docker/build/operator/helm
docker/build/operator/helm \
docker/build/readreplica-rotate

.PHONY: docker/name/org
docker/name/org:
Expand Down Expand Up @@ -184,6 +197,28 @@ docker/build/buildbase:
IMAGE=$(BUILDBASE_IMAGE) \
docker/build/image

.PHONY: docker/name/buildkit
docker/name/buildkit:
@echo "$(ORG)/$(BUILDKIT_IMAGE)"

.PHONY: docker/build/buildkit
## build buildkit image
docker/build/buildkit:
@make DOCKERFILE="$(ROOTDIR)/dockers/buildkit/Dockerfile" \
IMAGE=$(BUILDKIT_IMAGE) \
docker/build/image

.PHONY: docker/name/binfmt
docker/name/binfmt:
@echo "$(ORG)/$(BINFMT_IMAGE)"

.PHONY: docker/build/binfmt
## build binfmt image
docker/build/binfmt:
@make DOCKERFILE="$(ROOTDIR)/dockers/binfmt/Dockerfile" \
IMAGE=$(BINFMT_IMAGE) \
docker/build/image

.PHONY: docker/name/ci-container
docker/name/ci-container:
@echo "$(ORG)/$(CI_CONTAINER_IMAGE)"
Expand Down
2 changes: 1 addition & 1 deletion Makefile.d/e2e.mk
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ e2e/actions/run/job: \
kubectl wait -n kube-system --for=condition=Ready pod -l k8s-app=metrics-server --timeout=$(E2E_WAIT_FOR_START_TIMEOUT)
kubectl wait -n kube-system --for=condition=ContainersReady pod -l k8s-app=metrics-server --timeout=$(E2E_WAIT_FOR_START_TIMEOUT)
make k8s/vald/deploy \
HELM_VALUES=$(ROOTDIR)/.github/helm/values/values-lb.yaml
HELM_VALUES=$(ROOTDIR)/.github/helm/values/values-correction.yaml
sleep 3
kubectl wait --for=condition=Ready pod -l "app=$(LB_GATEWAY_IMAGE)" --timeout=$(E2E_WAIT_FOR_START_TIMEOUT)
kubectl wait --for=condition=ContainersReady pod -l "app=$(LB_GATEWAY_IMAGE)" --timeout=$(E2E_WAIT_FOR_START_TIMEOUT)
Expand Down
9 changes: 9 additions & 0 deletions Makefile.d/functions.mk
Original file line number Diff line number Diff line change
Expand Up @@ -118,12 +118,21 @@ define run-e2e-crud-test
-port=$(E2E_BIND_PORT) \
-dataset=$(ROOTDIR)/hack/benchmark/assets/dataset/$(E2E_DATASET_NAME) \
-insert-num=$(E2E_INSERT_COUNT) \
-correction-insert-num=$(E2E_INSERT_COUNT) \
-insert-num=$(E2E_INSERT_COUNT) \
-search-num=$(E2E_SEARCH_COUNT) \
-search-by-id-num=$(E2E_SEARCH_BY_ID_COUNT) \
-get-object-num=$(E2E_GET_OBJECT_COUNT) \
-update-num=$(E2E_UPDATE_COUNT) \
-upsert-num=$(E2E_UPSERT_COUNT) \
-remove-num=$(E2E_REMOVE_COUNT) \
-insert-from=$(E2E_INSERT_FROM) \
-update-from=$(E2E_UPDATE_FROM) \
-upsert-from=$(E2E_UPSERT_FROM) \
-remove-from=$(E2E_REMOVE_FROM) \
-search-from=$(E2E_SEARCH_FROM) \
-search-by-id-from=$(E2E_SEARCH_BY_ID_FROM) \
-get-object-from=$(E2E_GET_OBJECT_FROM) \
-wait-after-insert=$(E2E_WAIT_FOR_CREATE_INDEX_DURATION) \
-portforward=$(E2E_PORTFORWARD_ENABLED) \
-portforward-pod-name=$(E2E_TARGET_POD_NAME) \
Expand Down
Loading

0 comments on commit 2fc9958

Please sign in to comment.