Migratation to buf (#2236)

* migrate to buf Signed-off-by: Kosuke Morimoto <ksk@vdaas.org> * enable vtpool Signed-off-by: Kosuke Morimoto <ksk@vdaas.org> * migration of docs and swagger generation Signed-off-by: Kosuke Morimoto <ksk@vdaas.org> * revert removed files Signed-off-by: Kosuke Morimoto <ksk@vdaas.org> * update tool installation Signed-off-by: Kosuke Morimoto <ksk@vdaas.org> * update tool installation Signed-off-by: Kosuke Morimoto <ksk@vdaas.org> * fix Signed-off-by: Kosuke Morimoto <ksk@vdaas.org> * fix Signed-off-by: Kosuke Morimoto <ksk@vdaas.org> * format Signed-off-by: Kosuke Morimoto <ksk@vdaas.org> * go mod tidy Signed-off-by: Kosuke Morimoto <ksk@vdaas.org> * remove vtpool Signed-off-by: Kosuke Morimoto <ksk@vdaas.org> * add blank line Signed-off-by: Kosuke Morimoto <ksk@vdaas.org> * fix Signed-off-by: Kosuke Morimoto <ksk@vdaas.org> * fix Signed-off-by: Kosuke Morimoto <ksk@vdaas.org> * fix Signed-off-by: Kosuke Morimoto <ksk@vdaas.org> * update dependencies Signed-off-by: Kosuke Morimoto <ksk@vdaas.org> * formatting Signed-off-by: Kosuke Morimoto <ksk@vdaas.org> * style: format code with Gofumpt and Prettier This commit fixes the style issues introduced in e7a21ec according to the output from Gofumpt and Prettier. Details: #2236 --------- Signed-off-by: Kosuke Morimoto <ksk@vdaas.org> Co-authored-by: deepsource-autofix[bot] <62050782+deepsource-autofix[bot]@users.noreply.github.com> Co-authored-by: Yusuke Kato <kpango@vdaas.org> Co-authored-by: Yusuke Kadowaki <yusuke.kadowaki.1231@gmail.com>
vdaas · Dec 12, 2023 · d0d116f · d0d116f
1 parent e812aa2
commit d0d116f
Show file tree

Hide file tree

Showing 74 changed files with 2,254 additions and 2,599 deletions.
diff --git a/Makefile.d/functions.mk b/Makefile.d/functions.mk
@@ -30,37 +30,6 @@ define mkdir
 	mkdir -p $1
 endef
 
-define proto-code-gen
-	protoc \
-		$(PROTO_PATHS:%=-I %) \
-                --go_out=$(GOPATH)/src \
-		--plugin protoc-gen-go="$(GOBIN)/protoc-gen-go" \
-                --go-vtproto_out=$(GOPATH)/src \
-		--plugin protoc-gen-go-vtproto="$(GOBIN)/protoc-gen-go-vtproto" \
-                --go-vtproto_opt=features=grpc+marshal+unmarshal+size+equal+clone \
-		$1
-endef
-		# --go-grpc_out=$(GOPATH)/src \
-		# --plugin protoc-gen-go-grpc="${GOBIN}/protoc-gen-go-grpc" \
-                # --go-vtproto_opt=pool=$(ROOTDIR)/apis/proto/v1/payload.Object.Vector \
-                # --go-vtproto_opt=pool=$(ROOTDIR)/apis/proto/v1/payload.Insert.MultiRequest \
-                # --go-vtproto_opt=pool=$(ROOTDIR)/apis/proto/v1/payload.Insert.Request \
-                # --go-vtproto_opt=pool=$(ROOTDIR)/apis/proto/v1/payload.Object.Vector \
-                # --go-vtproto_opt=pool=$(ROOTDIR)/apis/proto/v1/payload.Object.Vectors \
-                # --go-vtproto_opt=pool=$(ROOTDIR)/apis/proto/v1/payload.Search.ObjectRequest \
-                # --go-vtproto_opt=pool=$(ROOTDIR)/apis/proto/v1/payload.Search.Request \
-                # --go-vtproto_opt=pool=$(ROOTDIR)/apis/proto/v1/payload.Update.MultiRequest \
-                # --go-vtproto_opt=pool=$(ROOTDIR)/apis/proto/v1/payload.Update.Request \
-                # --go-vtproto_opt=pool=$(ROOTDIR)/apis/proto/v1/payload.Upsert.MultiRequest \
-                # --go-vtproto_opt=pool=$(ROOTDIR)/apis/proto/v1/payload.Upsert.Request \
-
-define protoc-gen
-	protoc \
-		$(PROTO_PATHS:%=-I %) \
-		$2 \
-		$1
-endef
-
 define profile-web
 	go tool pprof -http=":6061" \
 		$1.bin \

diff --git a/Makefile.d/proto.mk b/Makefile.d/proto.mk
@@ -17,23 +17,13 @@
 ## build protobufs
 proto/all: \
 	proto/deps \
-	pbgo \
-	pbdoc \
-	swagger
-
-.PHONY: pbgo
-pbgo: $(PBGOS)
-
-.PHONY: swagger
-swagger: $(SWAGGERS)
-
-.PHONY: pbdoc
-pbdoc: $(PBDOCS)
+	proto/gen
 
 .PHONY: proto/clean
 ## clean proto artifacts
 proto/clean:
-	rm -rf apis/grpc apis/swagger apis/docs
+	find apis/grpc -name "*.pb.go" | xargs rm -f
+	rm -rf apis/swagger apis/docs
 
 .PHONY: proto/paths/print
 ## print proto paths
@@ -43,142 +33,28 @@ proto/paths/print:
 .PHONY: proto/deps
 ## install protobuf dependencies
 proto/deps: \
-	$(GOBIN)/protoc-gen-doc \
-	$(GOBIN)/protoc-gen-go \
-	$(GOBIN)/protoc-gen-go-grpc \
-	$(GOBIN)/protoc-gen-go-vtproto \
-	$(GOBIN)/protoc-gen-grpc-gateway \
-	$(GOBIN)/protoc-gen-swagger \
-	$(GOBIN)/protoc-gen-validate \
-	$(GOBIN)/prototool \
-	$(GOBIN)/swagger \
-	$(GOPATH)/src/github.com/envoyproxy/protoc-gen-validate \
-	$(GOPATH)/src/github.com/golang/protobuf \
-	$(GOPATH)/src/github.com/googleapis/googleapis \
-	$(GOPATH)/src/github.com/planetscale/vtprotobuf \
-	$(GOPATH)/src/github.com/protocolbuffers/protobuf \
-	$(GOPATH)/src/google.golang.org/genproto \
-	$(GOPATH)/src/google.golang.org/protobuf
+	$(GOBIN)/buf
 
 .PHONY: proto/clean/deps
 ## uninstall all protobuf dependencies
 proto/clean/deps:
-	rm -rf $(GOBIN)/protoc-gen-doc \
-	$(GOBIN)/protoc-gen-go \
-	$(GOBIN)/protoc-gen-go-grpc \
-	$(GOBIN)/protoc-gen-go-vtproto \
-	$(GOBIN)/protoc-gen-grpc-gateway \
-	$(GOBIN)/protoc-gen-swagger \
-	$(GOBIN)/protoc-gen-validate \
-	$(GOBIN)/prototool \
-	$(GOBIN)/swagger \
-	$(GOPATH)/src/github.com/envoyproxy/protoc-gen-validate \
-	$(GOPATH)/src/github.com/golang/protobuf \
-	$(GOPATH)/src/github.com/googleapis/googleapis \
-	$(GOPATH)/src/github.com/planetscale/vtprotobuf \
-	$(GOPATH)/src/github.com/protocolbuffers/protobuf \
-	$(GOPATH)/src/google.golang.org/genproto \
-	$(GOPATH)/src/google.golang.org/protobuf
-
-
-$(GOPATH)/src/github.com/protocolbuffers/protobuf:
-	git clone \
-		--depth 1 \
-		https://github.com/protocolbuffers/protobuf \
-		$(GOPATH)/src/github.com/protocolbuffers/protobuf
-
-$(GOPATH)/src/github.com/googleapis/googleapis:
-	git clone \
-		--depth 1 \
-		https://github.com/googleapis/googleapis \
-		$(GOPATH)/src/github.com/googleapis/googleapis
-
-$(GOPATH)/src/github.com/golang/protobuf:
-	git clone \
-		--depth 1 \
-		https://github.com/golang/protobuf \
-		$(GOPATH)/src/github.com/golang/protobuf
-
-$(GOPATH)/src/github.com/envoyproxy/protoc-gen-validate:
-	git clone \
-		--depth 1 \
-		https://github.com/envoyproxy/protoc-gen-validate \
-		$(GOPATH)/src/github.com/envoyproxy/protoc-gen-validate
-
-$(GOPATH)/src/google.golang.org/protobuf:
-	git clone \
-		--depth 1 \
-		https://go.googlesource.com/protobuf \
-		$(GOPATH)/src/google.golang.org/protobuf
-
-$(GOPATH)/src/github.com/planetscale/vtprotobuf:
-	git clone \
-		--depth 1 \
-		https://github.com/planetscale/vtprotobuf \
-		$(GOPATH)/src/github.com/planetscale/vtprotobuf
-
-$(GOPATH)/src/google.golang.org/genproto:
-	git clone \
-		--depth 1 \
-		https://github.com/googleapis/go-genproto \
-		$(GOPATH)/src/google.golang.org/genproto
-
-$(GOBIN)/protoc-gen-go:
-	$(call go-install, google.golang.org/protobuf/cmd/protoc-gen-go)
-
-$(GOBIN)/protoc-gen-go-grpc:
-	$(call go-install, google.golang.org/grpc/cmd/protoc-gen-go-grpc)
+	rm -rf $(GOBIN)/buf
 
-$(GOBIN)/protoc-gen-grpc-gateway:
-	$(call go-install, github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway)
+$(GOBIN)/buf:
+	$(call go-install, github.com/bufbuild/buf/cmd/buf)
 
-$(GOBIN)/protoc-gen-swagger:
-	$(call go-install, github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger)
+$(ROOTDIR)/apis/proto/v1/rpc/errdetails/error_details.proto:
+	curl -fsSL https://raw.githubusercontent.com/googleapis/googleapis/master/google/rpc/error_details.proto -o $(ROOTDIR)/apis/proto/v1/rpc/errdetails/error_details.proto
+	sed  -i -e "s/package google.rpc/package rpc.v1/" $(ROOTDIR)/apis/proto/v1/rpc/errdetails/error_details.proto
+	sed  -i -e "s%google.golang.org/genproto/googleapis/rpc/errdetails;errdetails%github.com/vdaas/vald/apis/grpc/v1/rpc/errdetails%" $(ROOTDIR)/apis/proto/v1/rpc/errdetails/error_details.proto
+	sed  -i -e "s/com.google.rpc/org.vdaas.vald.api.v1.rpc/" $(ROOTDIR)/apis/proto/v1/rpc/errdetails/error_details.proto
 
-$(GOBIN)/protoc-gen-validate:
-	$(call go-install, github.com/envoyproxy/protoc-gen-validate)
-
-$(GOBIN)/prototool:
-	$(call go-install, github.com/uber/prototool/cmd/prototool)
-
-$(GOBIN)/protoc-gen-doc:
-	$(call go-install, github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc)
-
-$(GOBIN)/protoc-gen-go-vtproto:
-	$(call go-install, github.com/planetscale/vtprotobuf/cmd/protoc-gen-go-vtproto)
-
-$(GOBIN)/swagger:
-	$(call go-install, github.com/go-swagger/go-swagger/cmd/swagger)
-
-$(ROOTDIR)/apis/proto/v1/rpc/error_details.proto:
-	curl -fsSL https://raw.githubusercontent.com/googleapis/googleapis/master/google/rpc/error_details.proto -o $(ROOTDIR)/apis/proto/v1/rpc/error_details.proto
-	sed  -i -e "s/package google.rpc/package rpc.v1/" $(ROOTDIR)/apis/proto/v1/rpc/error_details.proto
-	sed  -i -e "s%google.golang.org/genproto/googleapis/rpc/errdetails;errdetails%github.com/vdaas/vald/apis/grpc/v1/rpc/errdetails%" $(ROOTDIR)/apis/proto/v1/rpc/error_details.proto
-	sed  -i -e "s/com.google.rpc/org.vdaas.vald.api.v1.rpc/" $(ROOTDIR)/apis/proto/v1/rpc/error_details.proto
-
-$(PBGOS): \
+proto/gen: \
 	$(PROTOS) \
 	proto/deps
-	@$(call green, "generating pb.go files...")
-	$(call mkdir, $(dir $@))
-	$(call proto-code-gen, $(patsubst apis/grpc/%.pb.go,apis/proto/%.proto,$@))
+	@$(call green, "generating pb.go and swagger.json files and documents for API v1...")
+	buf generate
 	find $(ROOTDIR)/apis/grpc/* -name '*.go' | xargs sed -i -E "s%google.golang.org/grpc/codes%github.com/vdaas/vald/internal/net/grpc/codes%g"
 	find $(ROOTDIR)/apis/grpc/* -name '*.go' | xargs sed -i -E "s%google.golang.org/grpc/status%github.com/vdaas/vald/internal/net/grpc/status%g"
 	find $(ROOTDIR)/apis/grpc/* -name '*.go' | xargs sed -i -E "s%\"io\"%\"github.com/vdaas/vald/internal/io\"%g"
 	find $(ROOTDIR)/apis/grpc/* -name '*.go' | xargs sed -i -E "s%\"sync\"%\"github.com/vdaas/vald/internal/sync\"%g"
-
-$(SWAGGERS): \
-	$(PROTOS) \
-	proto/deps
-	@$(call green, "generating swagger.json files...")
-	$(call mkdir, $(dir $@))
-	$(call protoc-gen, $(patsubst apis/swagger/%.swagger.json,apis/proto/%.proto,$@), --swagger_out=json_names_for_fields=true:$(dir $@))
-
-$(PBDOCS): \
-	$(PROTOS) \
-	proto/deps
-
-apis/docs/v1/docs.md: $(PROTOS_V1)
-	@$(call green, "generating documents for API v1...")
-	$(call mkdir, $(dir $@))
-	$(call protoc-gen, $(PROTOS_V1), --plugin=protoc-gen-doc=$(GOBIN)/protoc-gen-doc --doc_opt=markdown$(COMMA)docs.md --doc_out=$(dir $@))
diff --git a/Makefile.d/tools.mk b/Makefile.d/tools.mk
@@ -66,15 +66,6 @@ $(BINDIR)/kubectl:
 	curl -L "https://dl.k8s.io/release/$(KUBECTL_VERSION)/bin/$(shell echo $(UNAME) | tr '[:upper:]' '[:lower:]')/$(subst x86_64,amd64,$(shell echo $(ARCH) | tr '[:upper:]' '[:lower:]'))/kubectl" -o $(BINDIR)/kubectl
 	chmod a+x $(BINDIR)/kubectl
 
-.PHONY: protobuf/install
-protobuf/install: $(BINDIR)/protoc
-
-$(BINDIR)/protoc:
-	curl -L "https://github.com/protocolbuffers/protobuf/releases/download/v$(PROTOBUF_VERSION)/protoc-$(PROTOBUF_VERSION)-$(subst darwin,osx,$(shell echo $(UNAME) | tr '[:upper:]' '[:lower:]'))-$(ARCH).zip" -o /tmp/protoc.zip
-	unzip -o /tmp/protoc.zip -d /usr/local bin/protoc
-	unzip -o /tmp/protoc.zip -d /usr/local 'include/*'
-	rm -f /tmp/protoc.zip
-
 .PHONY: textlint/install
 textlint/install:
 	npm install -g textlint textlint-rule-en-spell textlint-rule-prh textlint-rule-write-good