Add mirror gateway component (#1949)

* add mirror component Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * fix mirror dockerfile path Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * fix deepsource warning Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * fix insert rpc handler Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * deleted unused file Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * fix api resource name Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add mirror service and fix handler logic Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * fix values and helm template for new configuration Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * Format code with gofumpt and prettier * fix template rendering bug of mirror configmap Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * fix nil pointer error Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * bugfix to advertize different address Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * returns error when there is no other mirror gateways Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add info log Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * extract only mirror addresses Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * fix log variable Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add logging Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * bugfix mirror connection logic Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add remove api proxy implementation Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add error handling for register API Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * fix debug message Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * refactor discover logic Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * refactor Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add remove api Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * Format code with prettier and gofumpt * add remove api and remove api rollback logic Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * refactor Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add update rpc handler Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * refactor context variable for grpc method Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * fix trace span name Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add getObjects method Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * refactor update and remove handler Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * Apply suggestions from code review Co-authored-by: Kiichiro YUKAWA <kyukawa315@gmail.com> * add multi handler implementation Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * fix mirror client trace span Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * rename discover service to mirror service Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * fix build error of bidirectional stream Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add more error handling to search handler Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add debug comment Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add status code log Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add status code log Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add status code log Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * fix named return bug Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add debug log Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add debug command Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add debug command Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add debug command Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add force status wrap Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * execute upsert method to local cluster's lb Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add target debug log Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * fix delete unwrap of circuitbreaker Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * refactor error handling for new status parse function Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * Format code with prettier and gofumpt * refactor error handling Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * style: Format code with prettier and gofumpt * fix error handling for broadcast operation of getobject Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * fix error join bug of getobject Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * refactor Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * deleted unused code Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * fix status handling and add address to resource name Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * use vald client for update operation Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add rollback test for mirror crud Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * style: Format code with prettier and gofumpt * fix variable name Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * refactor mirror servie Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * deleted unused code Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * fix trace attribute and refactor variable name Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * add error handling when crud rpc fails Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * deleted unused value Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * make format Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * Revert "make format" This reverts commit f605563. * Apply suggestions from code review Co-authored-by: Kiichiro YUKAWA <kyukawa315@gmail.com> * apply suggestion Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> * bugfix and refactor mirror client Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> --------- Signed-off-by: hlts2 <hiroto.funakoshi.hiroto@gmail.com> Co-authored-by: deepsource-autofix[bot] <62050782+deepsource-autofix[bot]@users.noreply.github.com> Co-authored-by: Kiichiro YUKAWA <kyukawa315@gmail.com>
vdaas · Aug 15, 2023 · 0593c48 · 0593c48
1 parent 0debf63
commit 0593c48
Show file tree

Hide file tree

Showing 45 changed files with 12,439 additions and 2 deletions.
diff --git a/.github/workflows/dockers-gateway-mirror-image.yaml b/.github/workflows/dockers-gateway-mirror-image.yaml
@@ -0,0 +1,156 @@
+#
+# Copyright (C) 2019-2022 vdaas.org vald team <vald@vdaas.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+name: "Build docker image: gateway-mirror"
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - "*.*.*"
+      - "v*.*.*"
+      - "*.*.*-*"
+      - "v*.*.*-*"
+    paths:
+      - ".github/actions/docker-build/actions.yaml"
+      - ".github/workflows/dockers-gateway-mirror-image.yml"
+      - "go.mod"
+      - "go.sum"
+      - "internal/**"
+      - "!internal/**/*_test.go"
+      - "!internal/**/*_mock.go"
+      - "!internal/db/**"
+      - "!internal/k8s/**"
+      - "apis/grpc/**"
+      - "pkg/gateway/mirror/**"
+      - "cmd/gateway/mirror/**"
+      - "pkg/gateway/internal/**"
+      - "dockers/gateway/mirror/Dockerfile"
+      - "versions/GO_VERSION"
+  pull_request:
+    paths:
+      - ".github/actions/docker-build/actions.yaml"
+      - ".github/workflows/dockers-gateway-mirror-image.yml"
+      - "go.mod"
+      - "go.sum"
+      - "internal/**"
+      - "!internal/**/*_test.go"
+      - "!internal/**/*_mock.go"
+      - "!internal/db/**"
+      - "!internal/k8s/**"
+      - "apis/grpc/**"
+      - "pkg/gateway/mirror/**"
+      - "cmd/gateway/mirror/**"
+      - "pkg/gateway/internal/**"
+      - "dockers/gateway/mirror/Dockerfile"
+      - "versions/GO_VERSION"
+  pull_request_target:
+    paths:
+      - ".github/actions/docker-build/actions.yaml"
+      - ".github/workflows/dockers-gateway-mirror-image.yml"
+      - "go.mod"
+      - "go.sum"
+      - "internal/**"
+      - "!internal/**/*_test.go"
+      - "!internal/**/*_mock.go"
+      - "!internal/db/**"
+      - "!internal/k8s/**"
+      - "apis/grpc/**"
+      - "pkg/gateway/mirror/**"
+      - "cmd/gateway/nirror/**"
+      - "pkg/gateway/internal/**"
+      - "dockers/gateway/mirror/Dockerfile"
+      - "versions/GO_VERSION"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    strategy:
+      max-parallel: 4
+    runs-on: ubuntu-latest
+    if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event.pull_request.head.repo.fork == true && github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ci/approved')) }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: set git config
+        run: |
+          git config --global --add safe.directory ${GITHUB_WORKSPACE}
+      - name: Setup QEMU
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: all
+      - name: Setup Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          buildkitd-flags: "--debug"
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_PASS }}
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ secrets.PACKAGE_USER }}
+          password: ${{ secrets.PACKAGE_TOKEN }}
+      - name: Build and Publish
+        id: build_and_publish
+        uses: ./.github/actions/docker-build
+        with:
+          target: gateway-mirror
+          builder: ${{ steps.buildx.outputs.name }}
+      - name: Initialize CodeQL
+        if: startsWith( github.ref, 'refs/tags/')
+        uses: github/codeql-action/init@v2
+      - name: Run vulnerability scanner (table)
+        if: startsWith( github.ref, 'refs/tags/')
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}"
+          format: "table"
+      - name: Run vulnerability scanner (sarif)
+        if: startsWith( github.ref, 'refs/tags/')
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}"
+          format: "template"
+          template: "@/contrib/sarif.tpl"
+          output: "trivy-results.sarif"
+      - name: Upload Trivy scan results to Security tab
+        if: startsWith( github.ref, 'refs/tags/')
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: "trivy-results.sarif"
+  slack:
+    name: Slack notification
+    needs: build
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/main' || startsWith( github.ref, 'refs/tags/')
+    steps:
+      - uses: technote-space/workflow-conclusion-action@v2
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - uses: 8398a7/action-slack@v3
+        with:
+          author_name: vald-mirror-gateway image build
+          status: ${{ env.WORKFLOW_CONCLUSION }}
+          only_mention_fail: channel
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }}
diff --git a/Makefile b/Makefile
@@ -28,6 +28,7 @@ CI_CONTAINER_IMAGE              = $(NAME)-ci-container
 DEV_CONTAINER_IMAGE             = $(NAME)-dev-container
 DISCOVERER_IMAGE                = $(NAME)-discoverer-k8s
 FILTER_GATEWAY_IMAGE            = $(NAME)-filter-gateway
+MIRROR_GATEWAY_IMAGE            = $(NAME)-mirror-gateway
 HELM_OPERATOR_IMAGE             = $(NAME)-helm-operator
 LB_GATEWAY_IMAGE                = $(NAME)-lb-gateway
 LOADTEST_IMAGE                  = $(NAME)-loadtest

diff --git a/Makefile.d/build.mk b/Makefile.d/build.mk
@@ -177,6 +177,34 @@ cmd/gateway/filter/filter: \
 		$(dir $@)main.go
 	$@ -version
 
+cmd/gateway/mirror/mirror: \
+	$(GO_SOURCES_INTERNAL) \
+	$(PBGOS) \
+	$(shell find ./cmd/gateway/mirror -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') \
+	$(shell find ./pkg/gateway/mirror -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go')
+	CGO_ENABLED=0 \
+	GO111MODULE=on \
+	GOPRIVATE=$(GOPRIVATE) \
+	go build \
+		--ldflags "-w -extldflags=-static \
+		-X '$(GOPKG)/internal/info.Version=$(VERSION)' \
+		-X '$(GOPKG)/internal/info.GitCommit=$(GIT_COMMIT)' \
+		-X '$(GOPKG)/internal/info.BuildTime=$(DATETIME)' \
+		-X '$(GOPKG)/internal/info.GoVersion=$(GO_VERSION)' \
+		-X '$(GOPKG)/internal/info.GoOS=$(GOOS)' \
+		-X '$(GOPKG)/internal/info.GoArch=$(GOARCH)' \
+		-X '$(GOPKG)/internal/info.CGOEnabled=$${CGO_ENABLED}' \
+		-X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)' \
+		-buildid=" \
+		-mod=readonly \
+		-modcacherw \
+		-a \
+		-tags "osusergo netgo static_build" \
+		-trimpath \
+		-o $@ \
+		$(dir $@)main.go
+	$@ -version
+
 cmd/manager/index/index: \
 	$(GO_SOURCES_INTERNAL) \
 	$(PBGOS) \

diff --git a/Makefile.d/docker.mk b/Makefile.d/docker.mk
@@ -114,6 +114,21 @@ docker/build/gateway-filter:
 	    --build-arg DISTROLESS_IMAGE=$(DISTROLESS_IMAGE) \
 	    --build-arg DISTROLESS_IMAGE_TAG=$(DISTROLESS_IMAGE_TAG)
 
+.PHONY: docker/name/gateway-mirror
+docker/name/gateway-mirror:
+	@echo "$(ORG)/$(MIRROR_GATEWAY_IMAGE)"
+
+.PHONY: docker/build/gateway-mirror
+## build gateway-mirror image
+docker/build/gateway-mirror:
+	$(DOCKER) build \
+	    $(DOCKER_OPTS) \
+	    -f dockers/gateway/mirror/Dockerfile \
+	    -t $(ORG)/$(MIRROR_GATEWAY_IMAGE):$(TAG) . \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+	    --build-arg DISTROLESS_IMAGE=$(DISTROLESS_IMAGE) \
+	    --build-arg DISTROLESS_IMAGE_TAG=$(DISTROLESS_IMAGE_TAG)
+
 .PHONY: docker/name/manager-index
 docker/name/manager-index:
 	@echo "$(ORG)/$(MANAGER_INDEX_IMAGE)"

diff --git a/Makefile.d/k8s.mk b/Makefile.d/k8s.mk
@@ -15,6 +15,9 @@
 #
 
 JAEGER_OPERATOR_WAIT_DURATION := 0
+MIRROR01_NAMESPACE = vald-01
+MIRROR02_NAMESPACE = vald-02
+MIRROR03_NAMESPACE = vald-03
 
 .PHONY: k8s/manifest/clean
 ## clean k8s manifests
@@ -103,6 +106,33 @@ k8s/vald/delete:
 	kubectl delete -f $(TEMP_DIR)/vald/templates/agent
 	rm -rf $(TEMP_DIR)
 
+.PHONY: k8s/multi/vald/deploy
+## deploy multiple vald sample clusters to k8s
+k8s/multi/vald/deploy:
+	-@kubectl create ns $(MIRROR01_NAMESPACE)
+	-@kubectl create ns $(MIRROR02_NAMESPACE)
+	-@kubectl create ns $(MIRROR03_NAMESPACE)
+	helm install vald-cluster-01 charts/vald \
+		-f ./charts/vald/values/multi-vald/dev-vald-with-mirror.yaml \
+		-f ./charts/vald/values/multi-vald/dev-vald-01.yaml \
+	    -n $(MIRROR01_NAMESPACE)
+	helm install vald-cluster-02 charts/vald \
+		-f ./charts/vald/values/multi-vald/dev-vald-with-mirror.yaml \
+		-f ./charts/vald/values/multi-vald/dev-vald-02.yaml \
+	    -n $(MIRROR02_NAMESPACE)
+	helm install vald-cluster-03 charts/vald \
+		-f ./charts/vald/values/multi-vald/dev-vald-with-mirror.yaml \
+		-f ./charts/vald/values/multi-vald/dev-vald-03.yaml \
+		-n $(MIRROR03_NAMESPACE)
+
+.PHONY: k8s/multi/vald/delete
+## delete multiple vald sample clusters to k8s
+k8s/multi/vald/delete:
+	helm uninstall vald-cluster-01 -n vald-01
+	helm uninstall vald-cluster-02 -n vald-02
+	helm uninstall vald-cluster-03 -n vald-03
+	-@kubectl delete ns vald-01 vald-02 vald-03
+
 .PHONY: k8s/vald-helm-operator/deploy
 ## deploy vald-helm-operator to k8s
 k8s/vald-helm-operator/deploy:

diff --git a/apis/grpc/v1/vald/vald.go b/apis/grpc/v1/vald/vald.go
@@ -18,6 +18,7 @@
 package vald
 
 import (
+	"github.com/vdaas/vald/apis/grpc/v1/mirror"
 	grpc "google.golang.org/grpc"
 )
 
@@ -35,6 +36,11 @@ type ServerWithFilter interface {
 	FilterServer
 }
 
+type ServerWithMirror interface {
+	Server
+	mirror.MirrorServer
+}
+
 type UnimplementedValdServer struct {
 	UnimplementedInsertServer
 	UnimplementedUpdateServer
@@ -49,6 +55,11 @@ type UnimplementedValdServerWithFilter struct {
 	UnimplementedFilterServer
 }
 
+type UnimplementedValdServerWithMirror struct {
+	UnimplementedValdServer
+	mirror.UnimplementedMirrorServer
+}
+
 type Client interface {
 	InsertClient
 	UpdateClient
@@ -63,6 +74,11 @@ type ClientWithFilter interface {
 	FilterClient
 }
 
+type ClientWithMirror interface {
+	Client
+	mirror.MirrorClient
+}
+
 const PackageName = "vald.v1"
 
 const (
@@ -73,6 +89,7 @@ const (
 	RemoveRPCServiceName = "Remove"
 	ObjectRPCServiceName = "Object"
 	FilterRPCServiceName = "Filter"
+	MirrorRPCServiceName = "Mirror"
 )
 
 const (
@@ -124,6 +141,12 @@ const (
 	GetObjectRPCName        = "GetObject"
 	StreamGetObjectRPCName  = "StreamGetObject"
 	StreamListObjectRPCName = "StreamListObject"
+	ExistsRPCName           = "Exists"
+	GetObjectRPCName        = "GetObject"
+	StreamGetObjectRPCName  = "StreamGetObject"
+
+	RegisterRPCName  = "Register"
+	AdvertiseRPCName = "Advertise"
 )
 
 type client struct {
@@ -135,6 +158,11 @@ type client struct {
 	ObjectClient
 }
 
+type clientWithMirror struct {
+	Client
+	mirror.MirrorClient
+}
+
 func RegisterValdServer(s *grpc.Server, srv Server) {
 	RegisterInsertServer(s, srv)
 	RegisterUpdateServer(s, srv)
@@ -149,6 +177,11 @@ func RegisterValdServerWithFilter(s *grpc.Server, srv ServerWithFilter) {
 	RegisterFilterServer(s, srv)
 }
 
+func RegisterValdServerWithMirror(s *grpc.Server, srv ServerWithMirror) {
+	RegisterValdServer(s, srv)
+	mirror.RegisterMirrorServer(s, srv)
+}
+
 func NewValdClient(conn *grpc.ClientConn) Client {
 	return &client{
 		NewInsertClient(conn),
@@ -159,3 +192,10 @@ func NewValdClient(conn *grpc.ClientConn) Client {
 		NewObjectClient(conn),
 	}
 }
+
+func NewValdClientWithMirror(conn *grpc.ClientConn) ClientWithMirror {
+	return &clientWithMirror{
+		Client:       NewValdClient(conn),
+		MirrorClient: mirror.NewMirrorClient(conn),
+	}
+}