Add index-operator template implementation (#2375) (#2424)

* improve Backport Pull Request Title & Description Quality (#2373) * [bugfix] fix backport github value bypass failure (#2378) * [bugfix] fix backport github value bypass failure (#2381) * Add index operator boilerplate * Add k8s controller * Add pod and job controllers * Add annotations in the pod reconciler * Update auto indexing service configurations * Update WithOnReconcileFunc signature to include context * Refactor debugging log * Refactor reconcile funcs * add nolint * remove unused parameters * lint * Add dockerfile and build ci * remove params * Add index-operator image build command to Makefile * Add index operator k8s templates * Update helm schema * Update readme * Add internal/k8s as a trigger for images that depends on controller-runtime * remove comment * Update pkg/index/operator/service/operator.go * Update pkg/index/operator/service/operator.go * Refactor * Add build commands * Refactor --------- Signed-off-by: kpango <kpango@vdaas.org> Signed-off-by: Yusuke Kadowaki <yusuke.kadowaki.1231@gmail.com> Co-authored-by: Yusuke Kadowaki <yusuke.kadowaki.1231@gmail.com> Co-authored-by: Yusuke Kato <kpango@vdaas.org> Co-authored-by: Kosuke Morimoto <ksk@vdaas.org>
vdaas · Feb 27, 2024 · 012a3d3 · 012a3d3
1 parent 483699f
commit 012a3d3
Show file tree

Hide file tree

Showing 26 changed files with 4,171 additions and 951 deletions.
diff --git a/.github/workflows/dockers-agent-ngt-image.yml b/.github/workflows/dockers-agent-ngt-image.yml
@@ -34,7 +34,7 @@ on:
       - "internal/**"
       - "!internal/**/*_test.go"
       - "!internal/db/**"
-      - "!internal/k8s/**"
+      - "internal/k8s/**"
       - "apis/grpc/**"
       - "pkg/agent/core/ngt/**"
       - "cmd/agent/core/ngt/**"
@@ -51,7 +51,7 @@ on:
       - "internal/**"
       - "!internal/**/*_test.go"
       - "!internal/db/**"
-      - "!internal/k8s/**"
+      - "internal/k8s/**"
       - "apis/grpc/**"
       - "pkg/agent/core/ngt/**"
       - "cmd/agent/core/ngt/**"
@@ -68,7 +68,7 @@ on:
       - "internal/**"
       - "!internal/**/*_test.go"
       - "!internal/db/**"
-      - "!internal/k8s/**"
+      - "internal/k8s/**"
       - "apis/grpc/**"
       - "pkg/agent/core/ngt/**"
       - "cmd/agent/core/ngt/**"

diff --git a/.github/workflows/dockers-index-operator.yml b/.github/workflows/dockers-index-operator.yml
@@ -0,0 +1,77 @@
+#
+# Copyright (C) 2019-2024 vdaas.org vald team <vald@vdaas.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+name: "Build docker image: index-operator"
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - "*.*.*"
+      - "v*.*.*"
+      - "*.*.*-*"
+      - "v*.*.*-*"
+    paths:
+      - ".github/actions/docker-build/actions.yaml"
+      - ".github/workflows/dockers-index-operator.yml"
+      - "go.mod"
+      - "go.sum"
+      - "internal/**"
+      - "!internal/**/*_test.go"
+      - "!internal/db/**"
+      - "internal/k8s/**"
+      - "apis/grpc/**"
+      - "pkg/index/operator/**"
+      - "cmd/index/operator/**"
+      - "dockers/index/operator/Dockerfile"
+      - "versions/GO_VERSION"
+  pull_request:
+    paths:
+      - ".github/actions/docker-build/actions.yaml"
+      - ".github/workflows/_docker-image.yaml"
+      - ".github/workflows/dockers-index-operator.yml"
+      - "go.mod"
+      - "go.sum"
+      - "internal/**"
+      - "!internal/**/*_test.go"
+      - "!internal/db/**"
+      - "internal/k8s/**"
+      - "apis/grpc/**"
+      - "pkg/index/operator/**"
+      - "cmd/index/operator/**"
+      - "dockers/index/operator/Dockerfile"
+      - "versions/GO_VERSION"
+  pull_request_target:
+    paths:
+      - ".github/actions/docker-build/actions.yaml"
+      - ".github/workflows/_docker-image.yaml"
+      - ".github/workflows/dockers-index-operator.yml"
+      - "go.mod"
+      - "go.sum"
+      - "internal/**"
+      - "!internal/**/*_test.go"
+      - "!internal/db/**"
+      - "internal/k8s/**"
+      - "apis/grpc/**"
+      - "pkg/index/operator/**"
+      - "cmd/index/operator/**"
+      - "dockers/index/operator/Dockerfile"
+      - "versions/GO_VERSION"
+jobs:
+  build:
+    uses: ./.github/workflows/_docker-image.yaml
+    with:
+      target: index-operator
+    secrets: inherit
diff --git a/.github/workflows/dockers-readreplica-rotate.yml b/.github/workflows/dockers-readreplica-rotate.yml
@@ -31,7 +31,7 @@ on:
       - "internal/**"
       - "!internal/**/*_test.go"
       - "!internal/db/**"
-      - "!internal/k8s/**"
+      - "internal/k8s/**"
       - "apis/grpc/**"
       - "pkg/index/job/readreplica/rotate/**"
       - "cmd/index/job/readreplica/rotate/**"
@@ -47,7 +47,7 @@ on:
       - "internal/**"
       - "!internal/**/*_test.go"
       - "!internal/db/**"
-      - "!internal/k8s/**"
+      - "internal/k8s/**"
       - "apis/grpc/**"
       - "pkg/index/job/readreplica/rotate/**"
       - "cmd/index/job/readreplica/rotate/**"
@@ -63,7 +63,7 @@ on:
       - "internal/**"
       - "!internal/**/*_test.go"
       - "!internal/db/**"
-      - "!internal/k8s/**"
+      - "internal/k8s/**"
       - "apis/grpc/**"
       - "pkg/index/job/readreplica/rotate/**"
       - "cmd/index/job/readreplica/rotate/**"

diff --git a/Makefile b/Makefile
@@ -35,6 +35,7 @@ LOADTEST_IMAGE                  = $(NAME)-loadtest
 INDEX_CORRECTION_IMAGE          = $(NAME)-index-correction
 INDEX_CREATION_IMAGE            = $(NAME)-index-creation
 INDEX_SAVE_IMAGE                = $(NAME)-index-save
+INDEX_OPERATOR_IMAGE            = $(NAME)-index-operator
 READREPLICA_ROTATE_IMAGE        = $(NAME)-readreplica-rotate
 MANAGER_INDEX_IMAGE             = $(NAME)-manager-index
 BENCHMARK_JOB_IMAGE             = $(NAME)-benchmark-job

diff --git a/Makefile.d/build.mk b/Makefile.d/build.mk
@@ -30,7 +30,8 @@ binary/build: \
 	cmd/index/job/save/index-save \
 	cmd/manager/index/index \
 	cmd/tools/benchmark/job/job \
-	cmd/tools/benchmark/operator/operator
+	cmd/tools/benchmark/operator/operator \
+	cmd/index/operator/index-operator
 
 
 cmd/agent/core/ngt/ngt: \
@@ -83,6 +84,10 @@ cmd/index/job/readreplica/rotate/readreplica-rotate:
 	$(eval CGO_ENABLED = 0)
 	$(call go-build,index/job/readreplica/rotate,,-static,,,$@)
 
+cmd/index/operator/index-operator:
+	$(eval CGO_ENABLED = 0)
+	$(call go-build,index/operator,,-static,,,$@)
+
 cmd/tools/benchmark/job/job:
 	$(call go-build,tools/benchmark/job,-linkmode 'external',-static -fPIC -pthread -fopenmp -std=gnu++20 -lhdf5 -lhdf5_hl -lm -ldl, cgo,$(HDF5_VERSION),$@)
 
@@ -104,7 +109,8 @@ binary/build/zip: \
 	artifacts/vald-lb-gateway-$(GOOS)-$(GOARCH).zip \
 	artifacts/vald-manager-index-$(GOOS)-$(GOARCH).zip \
 	artifacts/vald-mirror-gateway-$(GOOS)-$(GOARCH).zip \
-	artifacts/vald-readreplica-rotate-$(GOOS)-$(GOARCH).zip
+	artifacts/vald-readreplica-rotate-$(GOOS)-$(GOARCH).zip \
+	artifacts/vald-index-operator-$(GOOS)-$(GOARCH).zip
 
 artifacts/vald-agent-ngt-$(GOOS)-$(GOARCH).zip: cmd/agent/core/ngt/ngt
 	$(call mkdir, $(dir $@))
@@ -161,3 +167,7 @@ artifacts/vald-index-save-$(GOOS)-$(GOARCH).zip: cmd/index/job/save/index-save
 artifacts/vald-readreplica-rotate-$(GOOS)-$(GOARCH).zip: cmd/index/job/readreplica/rotate/readreplica-rotate
 	$(call mkdir, $(dir $@))
 	zip --junk-paths $@ $<
+
+artifacts/vald-index-operator-$(GOOS)-$(GOARCH).zip: cmd/index/operator/index-operator
+	$(call mkdir, $(dir $@))
+	zip --junk-paths $@ $<
diff --git a/Makefile.d/docker.mk b/Makefile.d/docker.mk
@@ -242,6 +242,17 @@ docker/build/index-save:
 		IMAGE=$(INDEX_SAVE_IMAGE) \
 		docker/build/image
 
+.PHONY: docker/name/index-operator
+docker/name/index-operator:
+	@echo "$(ORG)/$(INDEX_OPERATOR_IMAGE)"
+
+.PHONY: docker/build/index-operator
+## build index-operator image
+docker/build/index-operator:
+	@make DOCKERFILE="$(ROOTDIR)/dockers/index/operator/Dockerfile" \
+		IMAGE=$(INDEX_OPERATOR_IMAGE) \
+		docker/build/image
+
 .PHONY: docker/name/readreplica-rotate
 docker/name/readreplica-rotate:
 	@echo "$(ORG)/$(READREPLICA_ROTATE_IMAGE)"

diff --git a/Makefile.d/k8s.mk b/Makefile.d/k8s.mk
@@ -47,6 +47,7 @@ k8s/manifest/update: \
 	mv $(TEMP_DIR)/vald/templates/discoverer k8s/discoverer
 	mv $(TEMP_DIR)/vald/templates/gateway k8s/gateway
 	mv $(TEMP_DIR)/vald/templates/manager/index k8s/manager/index
+	mv $(TEMP_DIR)/vald/templates/index/operator k8s/index/operator
 	mv $(TEMP_DIR)/vald/templates/index/job/correction k8s/index/job/correction
 	mv $(TEMP_DIR)/vald/templates/index/job/creation k8s/index/job/creation
 	mv $(TEMP_DIR)/vald/templates/index/job/save k8s/index/job/save
@@ -120,6 +121,7 @@ k8s/vald/deploy:
 	    --set manager.index.image.repository=$(CRORG)/$(MANAGER_INDEX_IMAGE) \
 	    --set manager.index.creator.image.repository=$(CRORG)/$(INDEX_CREATION_IMAGE) \
 	    --set manager.index.saver.image.repository=$(CRORG)/$(INDEX_SAVE_IMAGE) \
+		--set manager.index.operator.image.repository=$(CRORG)/$(INDEX_OPERATOR_IMAGE) \
 	    $(HELM_EXTRA_OPTIONS) \
         --include-crds \
 	    --output-dir $(TEMP_DIR) \
@@ -138,6 +140,7 @@ k8s/vald/deploy:
 	kubectl apply -f $(TEMP_DIR)/vald/templates/index/job/creation || true
 	kubectl apply -f $(TEMP_DIR)/vald/templates/index/job/save || true
 	kubectl apply -f $(TEMP_DIR)/vald/templates/index/job/readreplica/rotate || true
+	kubectl apply -f $(TEMP_DIR)/vald/templates/index/operator || true
 	rm -rf $(TEMP_DIR)
 	kubectl get pods -o jsonpath="{.items[*].spec.containers[*].image}" | tr " " "\n"
 
@@ -154,24 +157,26 @@ k8s/vald/delete:
 	    --set gateway.lb.image.repository=$(CRORG)/$(LB_GATEWAY_IMAGE) \
 	    --set gateway.mirror.image.repository=$(CRORG)/$(MIRROR_GATEWAY_IMAGE) \
 	    --set manager.index.image.repository=$(CRORG)/$(MANAGER_INDEX_IMAGE) \
+		--set manager.index.operator.image.repository=$(CRORG)/$(INDEX_OPERATOR_IMAGE) \
         --include-crds \
 	    --output-dir $(TEMP_DIR) \
 	    charts/vald
-	kubectl delete -f $(TEMP_DIR)/vald/templates/gateway/mirror
-	kubectl delete -f $(TEMP_DIR)/vald/templates/index/job/readreplica/rotate
-	kubectl delete -f $(TEMP_DIR)/vald/templates/index/job/save
-	kubectl delete -f $(TEMP_DIR)/vald/templates/index/job/creation
-	kubectl delete -f $(TEMP_DIR)/vald/templates/index/job/correction
-	kubectl delete -f $(TEMP_DIR)/vald/templates/index/job/creation
-	kubectl delete -f $(TEMP_DIR)/vald/templates/index/job/save
-	kubectl delete -f $(TEMP_DIR)/vald/templates/gateway
-	kubectl delete -f $(TEMP_DIR)/vald/templates/gateway/lb
-	kubectl delete -f $(TEMP_DIR)/vald/templates/manager/index
-	kubectl delete -f $(TEMP_DIR)/vald/templates/discoverer
+	kubectl delete -f $(TEMP_DIR)/vald/templates/gateway/mirror || true
+	kubectl delete -f $(TEMP_DIR)/vald/templates/index/operator || true
+	kubectl delete -f $(TEMP_DIR)/vald/templates/index/job/readreplica/rotate || true
+	kubectl delete -f $(TEMP_DIR)/vald/templates/index/job/save || true
+	kubectl delete -f $(TEMP_DIR)/vald/templates/index/job/creation || true
+	kubectl delete -f $(TEMP_DIR)/vald/templates/index/job/correction || true
+	kubectl delete -f $(TEMP_DIR)/vald/templates/index/job/creation || true
+	kubectl delete -f $(TEMP_DIR)/vald/templates/index/job/save || true
+	kubectl delete -f $(TEMP_DIR)/vald/templates/gateway || true
+	kubectl delete -f $(TEMP_DIR)/vald/templates/gateway/lb || true
+	kubectl delete -f $(TEMP_DIR)/vald/templates/manager/index || true
+	kubectl delete -f $(TEMP_DIR)/vald/templates/discoverer || true
 	kubectl delete -f $(TEMP_DIR)/vald/templates/agent/readreplica || true
 	kubectl delete -f $(TEMP_DIR)/vald/templates/agent/ngt || true
-	kubectl delete -f $(TEMP_DIR)/vald/templates/agent
-	kubectl delete -f $(TEMP_DIR)/vald/crds
+	kubectl delete -f $(TEMP_DIR)/vald/templates/agent || true
+	kubectl delete -f $(TEMP_DIR)/vald/crds || true
 	rm -rf $(TEMP_DIR)
 
 .PHONY: k8s/multi/vald/deploy