[ENG-447] protobuf workspaces migration (#725)

* retooling protobuf

* fixed buf generator to work properly

* fix protocgen so make protocgen works, proto files generated

* added statik, changelog

* build proto files when change

* split up docs for correct generation

* commit removed file

* deprecation(proto): replace go get with go install for rakyll/statik

Co-authored-by: Federico Kunze Küllmer <31522760+fedekunze@users.noreply.github.com>
Co-authored-by: Daniel Burckhardt <daniel.m.burckhardt@gmail.com>

.github/workflows/proto.yml

@@ -7,6 +7,22 @@ on:
       - "proto/**"
 
 jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: technote-space/get-diff-action@v6.1.0
+        id: git_diff
+        with:
+          PATTERNS: |
+            **/**.proto
+            **/buf.yaml
+            buf.work.yaml
+            buf.gen.yaml
+      - run: |
+          make proto-gen
+        if: env.GIT_DIFF
+
   lint:
     runs-on: ubuntu-latest
     timeout-minutes: 5

CHANGELOG.md

@@ -37,6 +37,10 @@ Ref: https://keepachangelog.com/en/1.0.0/
 
 ## Unreleased
 
+### Improvements
+
+- (build) [\#725](https://github.com/evmos/evmos/pull/725) Migrate Protobuf code generator to [Protobuf Workspaces](https://docs.buf.build/reference/workspaces)
+
 ### Bug Fixes
 
 - (inflation) [\#748](https://github.com/evmos/evmos/pull/748) Remove overcounted epochs from `skippedEpochs` value in store

Makefile

@@ -457,10 +457,6 @@ proto-gen:
 	@echo "Generating Protobuf files"
 	$(DOCKER) run --rm -v $(CURDIR):/workspace --workdir /workspace tendermintdev/sdk-proto-gen sh ./scripts/protocgen.sh
 
-proto-swagger-gen:
-	@echo "Generating Protobuf Swagger"
-	@./scripts/protoc-swagger-gen.sh
-
 proto-format:
 	@echo "Formatting Protobuf files"
 	find ./ -not -path "./third_party/*" -name *.proto -exec clang-format -i {} \;

buf.gen.proto.yaml

@@ -0,0 +1,19 @@
+version: v1
+
+plugins:
+
+  - name: gocosmos
+    out: .
+    opt:
+      - plugins=interfacetype+grpc
+
+  - name: grpc-gateway
+    out: .
+    opt:
+      - logtostderr=true
+
+  - name: doc
+    out: ./docs/protocol
+    opt:
+      - ./docs/protodoc-markdown.tmpl,proto-docs.md
+    strategy: all

buf.gen.swagger.yaml

@@ -0,0 +1,21 @@
+version: v1
+
+plugins:
+
+  - name: gocosmos
+    out: .
+    opt:
+      - plugins=interfacetype+grpc
+
+  - name: grpc-gateway
+    out: .
+    opt:
+      - logtostderr=true
+
+  - name: swagger
+    out: ./tmp-swagger-gen
+    opt:
+      - logtostderr=true
+      - fqn_for_swagger_name=true
+      - simple_operation_ids=true
+    strategy: all

scripts/protocgen.sh

@@ -18,28 +18,39 @@ protoc_gen_doc() {
 protoc_gen_gocosmos
 protoc_gen_doc
 
-proto_dirs=$(find ./proto -path -prune -o -name '*.proto' -print0 | xargs -0 -n1 dirname | sort | uniq)
-# TODO: migrate to `buf build`
+# need to install statik on the docker image
+go install github.com/rakyll/statik
+
+# create temporary folder to store intermediate results from `buf build` + `buf generate`
+mkdir -p ./tmp-swagger-gen
+
+# build .proto files and generate code for the proto/ directory
+buf build proto
+buf generate proto --template buf.gen.proto.yaml
+
+# create additional swagger files on an individual basis  w/ `buf build` and `buf generate` (needed for `swagger-combine`)
+proto_dirs=$(find ./proto ./third_party/proto -path -prune -o -name '*.proto' -print0 | xargs -0 -n1 dirname | sort | uniq)
 for dir in $proto_dirs; do
-  buf alpha protoc \
-  -I "proto" \
-  -I "third_party/proto" \
-  --gocosmos_out=plugins=interfacetype+grpc,\
-Mgoogle/protobuf/any.proto=github.com/cosmos/cosmos-sdk/codec/types:. \
-  --grpc-gateway_out=logtostderr=true:. \
-  $(find "${dir}" -maxdepth 1 -name '*.proto')
 
+  # generate swagger files (filter query files)
+  query_file=$(find "${dir}" -maxdepth 1 \( -name 'query.proto' -o -name 'service.proto' \))
+  if [[ ! -z "$query_file" ]]; then
+    buf build --path "$query_file"
+    buf generate --path "$query_file" --template buf.gen.swagger.yaml
+  fi
 done
 
-# command to generate docs using protoc-gen-doc
-# TODO: migrate to `buf build`
-buf alpha protoc \
--I "proto" \
--I "third_party/proto" \
---doc_out=./docs/protocol \
---doc_opt=./docs/protodoc-markdown.tmpl,proto-docs.md \
-$(find "$(pwd)/proto" -maxdepth 5 -name '*.proto')
-
-# move proto files to the right places
+# move resulting files to the right places
 cp -r github.com/evmos/evmos/v*/x/* x/
 rm -rf github.com
+
+# combine swagger files
+# uses nodejs package `swagger-combine`.
+# all the individual swagger files need to be configured in `config.json` for merging
+swagger-combine ./client/docs/config.json -o ./client/docs/swagger-ui/swagger.yaml -f yaml --continueOnConflictingPaths true --includeDefinitions true
+
+# clean swagger files
+rm -rf ./tmp-swagger-gen
+
+# generate binary for static server (use -f flag to replace current binary)
+statik -f -src=./client/docs/swagger-ui -dest=./client/docs

third_party/proto/google/protobuf/any.proto

@@ -35,7 +35,7 @@ package google.protobuf;
 import "gogoproto/gogo.proto";
 
 option csharp_namespace = "Google.Protobuf.WellKnownTypes";
-option go_package = "types";
+option go_package = "github.com/cosmos/cosmos-sdk/codec/types";
 option java_package = "com.google.protobuf";
 option java_outer_classname = "AnyProto";
 option java_multiple_files = true;