diff --git a/MANIFEST.in b/MANIFEST.in index fa15133f4..0b55e5636 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -9,5 +9,6 @@ include setup.py include setup.cfg include LICENSE include MANIFEST.in +include *.so recursive-exclude examples *~ *.pyc \.* diff --git a/go/protopace/.gitignore b/go/protopace/.gitignore new file mode 100644 index 000000000..6f72f8926 --- /dev/null +++ b/go/protopace/.gitignore @@ -0,0 +1,25 @@ +# If you prefer the allow list template instead of the deny list, see community template: +# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore +# +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# Go workspace file +go.work +go.work.sum + +# env file +.env diff --git a/go/protopace/Makefile b/go/protopace/Makefile new file mode 100644 index 000000000..93204a18e --- /dev/null +++ b/go/protopace/Makefile @@ -0,0 +1,101 @@ +# Change these variables as necessary. +MAIN_PACKAGE_PATH := . +BINARY_NAME := protopace +BUILD_DIR := ../../karapace/protobuf/protopace/bin + +# ==================================================================================== # +# HELPERS +# ==================================================================================== # + +## help: print this help message +.PHONY: help +help: + @echo 'Usage:' + @sed -n 's/^##//p' ${MAKEFILE_LIST} | column -t -s ':' | sed -e 's/^/ /' + +.PHONY: confirm +confirm: + @echo -n 'Are you sure? [y/N] ' && read ans && [ $${ans:-N} = y ] + +.PHONY: no-dirty +no-dirty: + git diff --exit-code + + +# ==================================================================================== # +# QUALITY CONTROL +# ==================================================================================== # + +## tidy: format code and tidy modfile +.PHONY: tidy +tidy: + go fmt ./... + go mod tidy -v + +## audit: run quality control checks +.PHONY: audit +audit: + go mod verify + go vet ./... + go run honnef.co/go/tools/cmd/staticcheck@latest -checks=all,-ST1000,-U1000 ./... + go run golang.org/x/vuln/cmd/govulncheck@latest ./... + go test -race -buildvcs -vet=off ./... + + +# ==================================================================================== # +# DEVELOPMENT +# ==================================================================================== # + +## test: run all tests +.PHONY: test +test: + go test -v -race -buildvcs ./... + +## test/cover: run all tests and display coverage +.PHONY: test/cover +test/cover: + go test -v -race -buildvcs -coverprofile=/tmp/coverage.out ./... + go tool cover -html=/tmp/coverage.out + +## build: build the application +.PHONY: build +build: + # Include additional build steps, like TypeScript, SCSS or Tailwind compilation here... + go build -o=/tmp/bin/${BINARY_NAME} ${MAIN_PACKAGE_PATH} + +## run: run the application +.PHONY: run +run: build + /tmp/bin/${BINARY_NAME} + +## run/live: run the application with reloading on file changes +.PHONY: run/live +run/live: + go run github.com/cosmtrek/air@v1.43.0 \ + --build.cmd "make build" --build.bin "/tmp/bin/${BINARY_NAME}" --build.delay "100" \ + --build.exclude_dir "" \ + --build.include_ext "go, tpl, tmpl, html, css, scss, js, ts, sql, jpeg, jpg, gif, png, bmp, svg, webp, ico" \ + --misc.clean_on_exit "true" + + +# ==================================================================================== # +# OPERATIONS +# ==================================================================================== # + +## release: cross-compile to build dir on mac +.PHONY: release +release: + CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build -ldflags='-s -w' -o=${BUILD_DIR}/${BINARY_NAME}-darwin-amd64.so -buildmode=c-shared ${MAIN_PACKAGE_PATH} + CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build -ldflags='-s -w' -o=${BUILD_DIR}/${BINARY_NAME}-darwin-arm64.so -buildmode=c-shared ${MAIN_PACKAGE_PATH} + + docker run --rm -v "${PWD}":/usr/src/myapp -w /usr/src/myapp --platform=linux/amd64 golang:1.22 env GOOS=linux GOARCH=amd64 CGO_ENABLED=1 go build -ldflags='-s -w' -buildmode=c-shared -o ${BINARY_NAME}-linux-amd64.so + cp ${BINARY_NAME}-linux-amd64.so ${BUILD_DIR}/${BINARY_NAME}-linux-amd64.so + cp ${BINARY_NAME}-linux-amd64.h ${BUILD_DIR}/${BINARY_NAME}-linux-amd64.h + rm ${BINARY_NAME}-linux-amd64.so + rm ${BINARY_NAME}-linux-amd64.h + + docker run --rm -v "${PWD}":/usr/src/myapp -w /usr/src/myapp --platform=linux/arm64 golang:1.22 env GOOS=linux GOARCH=arm64 CGO_ENABLED=1 go build -ldflags='-s -w' -buildmode=c-shared -o ${BINARY_NAME}-linux-arm64.so + cp ${BINARY_NAME}-linux-arm64.so ${BUILD_DIR}/${BINARY_NAME}-linux-arm64.so + cp ${BINARY_NAME}-linux-arm64.h ${BUILD_DIR}/${BINARY_NAME}-linux-arm64.h + rm ${BINARY_NAME}-linux-arm64.so + rm ${BINARY_NAME}-linux-arm64.h diff --git a/go/protopace/compatibility.go b/go/protopace/compatibility.go new file mode 100644 index 000000000..ee90df999 --- /dev/null +++ b/go/protopace/compatibility.go @@ -0,0 +1,41 @@ +package main + +import ( + "context" + + "github.com/Aiven-Open/karapace/go/protopace/schema" + + "github.com/bufbuild/buf/private/bufpkg/bufcheck/bufbreaking" + "github.com/bufbuild/buf/private/bufpkg/bufconfig" + "github.com/bufbuild/buf/private/pkg/tracing" + "go.uber.org/zap" +) + +func Check(schema schema.Schema, previousSchema schema.Schema) error { + handler := bufbreaking.NewHandler(zap.NewNop(), tracing.NopTracer) + ctx := context.Background() + image, err := schema.CompileBufImage() + if err != nil { + return err + } + previousImage, err := previousSchema.CompileBufImage() + if err != nil { + return err + } + checkConfig, _ := bufconfig.NewEnabledCheckConfig( + bufconfig.FileVersionV2, + nil, + []string{ + "FIELD_NO_DELETE", + "FILE_SAME_PACKAGE", + "FIELD_SAME_NAME", + "FIELD_SAME_JSON_NAME", + "FILE_NO_DELETE", + "ENUM_NO_DELETE", + }, + nil, + nil, + ) + config := bufconfig.NewBreakingConfig(checkConfig, false) + return handler.Check(ctx, config, previousImage, image) +} diff --git a/go/protopace/compatibility_test.go b/go/protopace/compatibility_test.go new file mode 100644 index 000000000..a9a5dd16e --- /dev/null +++ b/go/protopace/compatibility_test.go @@ -0,0 +1,34 @@ +package main + +import ( + "os" + "testing" + + s "github.com/Aiven-Open/karapace/go/protopace/schema" + "github.com/stretchr/testify/assert" +) + +func TestCompatibility(t *testing.T) { + assert := assert.New(t) + + data, _ := os.ReadFile("./fixtures/dependency.proto") + dependencySchema, err := s.FromString("my/awesome/customer/v1/nested_value.proto", string(data), nil) + assert.NoError(err) + assert.NotNil(dependencySchema) + + data, _ = os.ReadFile("./fixtures/test.proto") + testSchema, err := s.FromString("test.proto", string(data), []s.Schema{*dependencySchema}) + assert.NoError(err) + assert.NotNil(testSchema) + + data, _ = os.ReadFile("./fixtures/test_previous.proto") + previousSchema, err := s.FromString("test.proto", string(data), []s.Schema{*dependencySchema}) + assert.NoError(err) + assert.NotNil(previousSchema) + + err = Check(*testSchema, *testSchema) + assert.NoError(err) + + err = Check(*testSchema, *previousSchema) + assert.ErrorContains(err, "Field \"5\" with name \"foo\" on message \"EventValue\" changed type from \"string\" to \"int32\".") +} diff --git a/go/protopace/fixtures/dependency.proto b/go/protopace/fixtures/dependency.proto new file mode 100644 index 000000000..45b94a1de --- /dev/null +++ b/go/protopace/fixtures/dependency.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; +package my.awesome.customer.v1; + +message NestedValue { + string value = 1; +} + +enum Status { + UNKNOWN = 0; + ACTIVE = 1; + INACTIVE = 2; +} diff --git a/go/protopace/fixtures/test.proto b/go/protopace/fixtures/test.proto new file mode 100644 index 000000000..e973b579d --- /dev/null +++ b/go/protopace/fixtures/test.proto @@ -0,0 +1,30 @@ +syntax = "proto3"; + +package my.awesome.customer.v1; + +import "my/awesome/customer/v1/nested_value.proto"; +import "google/protobuf/timestamp.proto"; + +option ruby_package = "My::Awesome::Customer::V1"; +option csharp_namespace = "my.awesome.customer.V1"; +option go_package = "github.com/customer/api/my/awesome/customer/v1;dspv1"; +option java_multiple_files = true; +option java_outer_classname = "EventValueProto"; +option java_package = "com.my.awesome.customer.v1"; +option objc_class_prefix = "TDD"; +option php_metadata_namespace = "My\\Awesome\\Customer\\V1"; +option php_namespace = "My\\Awesome\\Customer\\V1"; + +message Local { + message NestedValue { + string foo = 1; + } +} + +message EventValue { + NestedValue nested_value = 1; + google.protobuf.Timestamp created_at = 2; + Status status = 3; + Local.NestedValue local_nested_value = 4; + int32 foo = 5; +} diff --git a/go/protopace/fixtures/test_previous.proto b/go/protopace/fixtures/test_previous.proto new file mode 100644 index 000000000..6ca7b7c06 --- /dev/null +++ b/go/protopace/fixtures/test_previous.proto @@ -0,0 +1,30 @@ +syntax = "proto3"; + +package my.awesome.customer.v1; + +import "my/awesome/customer/v1/nested_value.proto"; +import "google/protobuf/timestamp.proto"; + +option ruby_package = "My::Awesome::Customer::V1"; +option csharp_namespace = "my.awesome.customer.V1"; +option go_package = "github.com/customer/api/my/awesome/customer/v1;dspv1"; +option java_multiple_files = true; +option java_outer_classname = "EventValueProto"; +option java_package = "com.my.awesome.customer.v1"; +option objc_class_prefix = "TDD"; +option php_metadata_namespace = "My\\Awesome\\Customer\\V1"; +option php_namespace = "My\\Awesome\\Customer\\V1"; + +message Local { + message NestedValue { + string foo = 1; + } +} + +message EventValue { + NestedValue nested_value = 1; + google.protobuf.Timestamp created_at = 2; + Status status = 3; + Local.NestedValue local_nested_value = 4; + string foo = 5; +} diff --git a/go/protopace/formatter.go b/go/protopace/formatter.go new file mode 100644 index 000000000..cf61c82f3 --- /dev/null +++ b/go/protopace/formatter.go @@ -0,0 +1,2500 @@ +package main + +import ( + "errors" + "fmt" + "io" + "reflect" + "sort" + "strings" + "unicode" + "unicode/utf8" + + s "github.com/Aiven-Open/karapace/go/protopace/schema" + "github.com/bufbuild/protocompile/ast" + "github.com/bufbuild/protocompile/walk" + "go.uber.org/multierr" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" +) + +func Format(schema s.Schema) (s.Schema, error) { + all, err := schema.Compile() + if err != nil { + return schema, err + } + res := all[0] + + astNodeMapping := map[ast.Node]protoreflect.FullName{} + + walk.DescriptorProtos(res.FileDescriptorProto(), func(fn protoreflect.FullName, m proto.Message) error { + astNode := res.Node(m) + astNodeMapping[astNode] = fn + return nil + }) + + fieldTypeMapping := map[protoreflect.FullName]string{} + + walk.Descriptors(res, func(d protoreflect.Descriptor) error { + fd, ok := d.(protoreflect.FieldDescriptor) + if ok { + message := fd.Message() + if message != nil { + fieldTypeMapping[fd.FullName()] = string(message.FullName()) + return nil + } + enum := fd.Enum() + if enum != nil { + fieldTypeMapping[fd.FullName()] = string(enum.FullName()) + return nil + } + } + return nil + }) + writer := strings.Builder{} + fileNode := res.FileNode().(*ast.FileNode) + f := newFormatter(&writer, fileNode, astNodeMapping, fieldTypeMapping) + f.Run() + newSchema := schema + newSchema.Schema = writer.String() + return newSchema, nil +} + +type formatter struct { + writer io.Writer + fileNode *ast.FileNode + astNodeMapping map[ast.Node]protoreflect.FullName + fieldTypeMapping map[protoreflect.FullName]string + + // Used to adjust comments when we remove superfluous + // separators tp canonicalize message literals + overrideTrailingComments map[ast.Node]ast.Comments + + // Current level of indentation. + indent int + // The last character written to writer. + lastWritten rune + + // The last node written. This must be updated from all functions + // that write comments with a node. This flag informs how the next + // node's leading comments and whitespace should be written. + previousNode ast.Node + + // If true, a space will be written to the output unless the next character + // written is a newline (don't wait errant trailing spaces). + pendingSpace bool + // If true, the formatter is in the middle of printing compact options. + inCompactOptions bool + + // Track runes that open blocks/scopes and are expected to increase indention + // level. For example, when runes "{" "[" "(" ")" are written, the pending + // value is 2 (increment three times for "{" "[" "("; decrement once for ")"). + // If it's greater than zero at the end of a line, we call In() so that + // subsequent lines are indented. If it's less than zero at the end of a line, + // we call Out(). This minimizes the amount of explicit indent/unindent code + // that is needed and makes it less error-prone. + pendingIndent int + // If true, an inline node/sequence is being written. We treat whitespace a + // little differently for when blocks are printed inline vs. across multiple + // lines. So this flag informs the logic that makes those whitespace decisions. + inline bool + + // Records all errors that occur during the formatting process. Nearly any + // non-nil error represents a bug in the implementation. + err error +} + +// newFormatter returns a new formatter for the given file. +func newFormatter( + writer io.Writer, + fileNode *ast.FileNode, + astNodeMapping map[ast.Node]protoreflect.FullName, + fieldTypeMapping map[protoreflect.FullName]string, +) *formatter { + return &formatter{ + writer: writer, + fileNode: fileNode, + astNodeMapping: astNodeMapping, + fieldTypeMapping: fieldTypeMapping, + overrideTrailingComments: map[ast.Node]ast.Comments{}, + } +} + +// Run runs the formatter and writes the file's content to the formatter's writer. +func (f *formatter) Run() error { + f.writeFile() + return f.err +} + +// P prints a line to the generated output. +// +// This will emit a newline and proper indentation. If you do not +// want to emit a newline and want to write a raw string, use +// WriteString (which P calls). +// +// If strings.TrimSpace(elem) is empty, no indentation is produced. +func (f *formatter) P(elem string) { + if len(strings.TrimSpace(elem)) > 0 { + // We only want to write an indent if we're + // writing a non-empty string (not just a newline). + f.Indent(nil) + f.WriteString(elem) + } + f.WriteString("\n") + + if f.pendingIndent > 0 { + f.In() + } else if f.pendingIndent < 0 { + f.Out() + } + f.pendingIndent = 0 +} + +// Space adds a space to the generated output. +func (f *formatter) Space() { + f.pendingSpace = true +} + +// In increases the current level of indentation. +func (f *formatter) In() { + f.indent++ +} + +// Out reduces the current level of indentation. +func (f *formatter) Out() { + if f.indent <= 0 { + // Unreachable. + f.err = multierr.Append( + f.err, + errors.New("internal error: attempted to decrement indentation at zero"), + ) + return + } + f.indent-- +} + +// Indent writes the number of spaces associated +// with the current level of indentation. +func (f *formatter) Indent(nextNode ast.Node) { + // only indent at beginning of line + if f.lastWritten != '\n' { + return + } + indent := f.indent + if rn, ok := nextNode.(*ast.RuneNode); ok && indent > 0 { + if strings.ContainsRune("}])>", rn.Rune) { + indent-- + } + } + f.WriteString(strings.Repeat(" ", indent)) +} + +// WriteString writes the given element to the generated output. +// +// This will not write indentation or newlines. Use P if you +// want to emit identation or newlines. +func (f *formatter) WriteString(elem string) { + if f.pendingSpace { + f.pendingSpace = false + first, _ := utf8.DecodeRuneInString(elem) + + // We don't want "dangling spaces" before certain characters: + // newlines, commas, and semicolons. Also, when writing + // elements inline, we don't want spaces before close parens + // and braces. Similarly, we don't want extra/doubled spaces + // or dangling spaces after certain characters when printing + // inline, like open parens/braces. So only print the space + // if the previous and next character don't match above + // conditions. + + prevBlockList := "\x00 \t\n" + nextBlockList := "\n;," + if f.inline { + prevBlockList = "\x00 \t\n<[{(" + nextBlockList = "\n;,)]}>" + } + + if !strings.ContainsRune(prevBlockList, f.lastWritten) && + !strings.ContainsRune(nextBlockList, first) { + if _, err := f.writer.Write([]byte{' '}); err != nil { + f.err = multierr.Append(f.err, err) + return + } + } + } + if len(elem) == 0 { + return + } + f.lastWritten, _ = utf8.DecodeLastRuneInString(elem) + if _, err := f.writer.Write([]byte(elem)); err != nil { + f.err = multierr.Append(f.err, err) + } +} + +// SetPreviousNode sets the previously written node. This should +// be called in all of the comment writing functions. +func (f *formatter) SetPreviousNode(node ast.Node) { + f.previousNode = node +} + +// writeFile writes the file node. +func (f *formatter) writeFile() { + f.writeFileHeader() + f.writeFileTypes() + if f.fileNode.EOF != nil { + info := f.nodeInfo(f.fileNode.EOF) + f.writeMultilineComments(info.LeadingComments()) + } + if f.lastWritten != 0 && f.lastWritten != '\n' { + // If anything was written, we always conclude with + // a newline. + f.P("") + } +} + +// writeFileHeader writes the header of a .proto file. This includes the syntax, +// package, imports, and options (in that order). The imports and options are +// sorted. All other file elements are handled by f.writeFileTypes. +// +// For example, +// +// syntax = "proto3"; +// +// package acme.v1.weather; +// +// import "acme/payment/v1/payment.proto"; +// import "google/type/datetime.proto"; +// +// option cc_enable_arenas = true; +// option optimize_for = SPEED; +func (f *formatter) writeFileHeader() { + var ( + packageNode *ast.PackageNode + importNodes []*ast.ImportNode + optionNodes []*ast.OptionNode + ) + for _, fileElement := range f.fileNode.Decls { + switch node := fileElement.(type) { + case *ast.PackageNode: + packageNode = node + case *ast.ImportNode: + importNodes = append(importNodes, node) + case *ast.OptionNode: + optionNodes = append(optionNodes, node) + default: + continue + } + } + if f.fileNode.Syntax == nil && f.fileNode.Edition == nil && + packageNode == nil && importNodes == nil && optionNodes == nil { + // There aren't any header values, so we can return early. + return + } + editionNode := f.fileNode.Edition + if editionNode != nil { + f.writeEdition(editionNode) + } + if syntaxNode := f.fileNode.Syntax; syntaxNode != nil && editionNode == nil { + f.writeSyntax(syntaxNode) + } + if packageNode != nil { + f.writePackage(packageNode) + } + sort.Slice(importNodes, func(i, j int) bool { + iName := importNodes[i].Name.AsString() + jName := importNodes[j].Name.AsString() + // sort by public > None > weak + iOrder := importSortOrder(importNodes[i]) + jOrder := importSortOrder(importNodes[j]) + + if iName < jName { + return true + } + if iName > jName { + return false + } + if iOrder > jOrder { + return true + } + if iOrder < jOrder { + return false + } + + // put commented import first + return !f.importHasComment(importNodes[j]) + }) + for i, importNode := range importNodes { + if i == 0 && f.previousNode != nil && !f.leadingCommentsContainBlankLine(importNode) { + f.P("") + } + + // since the imports are sorted, this will skip write imports + // if they have appear before and dont have comment + if i > 0 && importNode.Name.AsString() == importNodes[i-1].Name.AsString() && + !f.importHasComment(importNode) { + continue + } + + f.writeImport(importNode, i > 0) + } + sort.Slice(optionNodes, func(i, j int) bool { + // The default options (e.g. cc_enable_arenas) should always + // be sorted above custom options (which are identified by a + // leading '('). + left := stringForOptionName(optionNodes[i].Name) + right := stringForOptionName(optionNodes[j].Name) + if strings.HasPrefix(left, "(") && !strings.HasPrefix(right, "(") { + // Prefer the default option on the right. + return false + } + if !strings.HasPrefix(left, "(") && strings.HasPrefix(right, "(") { + // Prefer the default option on the left. + return true + } + // Both options are custom, so we defer to the standard sorting. + return left < right + }) + for i, optionNode := range optionNodes { + if i == 0 && f.previousNode != nil && !f.leadingCommentsContainBlankLine(optionNode) { + f.P("") + } + f.writeFileOption(optionNode, i > 0) + } +} + +// writeFileTypes writes the types defined in a .proto file. This includes the messages, enums, +// services, etc. All other elements are ignored since they are handled by f.writeFileHeader. +func (f *formatter) writeFileTypes() { + for i, fileElement := range f.fileNode.Decls { + switch node := fileElement.(type) { + case *ast.PackageNode, *ast.OptionNode, *ast.ImportNode, *ast.EmptyDeclNode: + // These elements have already been written by f.writeFileHeader. + continue + default: + info := f.nodeInfo(node) + wantNewline := f.previousNode != nil && (i == 0 || info.LeadingComments().Len() > 0) + if wantNewline && !f.leadingCommentsContainBlankLine(node) { + f.P("") + } + f.writeNode(node) + } + } +} + +// writeSyntax writes the syntax. +// +// For example, +// +// syntax = "proto3"; +func (f *formatter) writeSyntax(syntaxNode *ast.SyntaxNode) { + f.writeStart(syntaxNode.Keyword) + f.Space() + f.writeInline(syntaxNode.Equals) + f.Space() + f.writeInline(syntaxNode.Syntax) + f.writeLineEnd(syntaxNode.Semicolon) +} + +// writeEdition writes the edition. +// +// For example, +// +// edition = "2023"; +func (f *formatter) writeEdition(editionNode *ast.EditionNode) { + f.writeStart(editionNode.Keyword) + f.Space() + f.writeInline(editionNode.Equals) + f.Space() + f.writeInline(editionNode.Edition) + f.writeLineEnd(editionNode.Semicolon) +} + +// writePackage writes the package. +// +// For example, +// +// package acme.weather.v1; +func (f *formatter) writePackage(packageNode *ast.PackageNode) { + f.writeStart(packageNode.Keyword) + f.Space() + f.writeInline(packageNode.Name) + f.writeLineEnd(packageNode.Semicolon) +} + +// writeImport writes an import statement. +// +// For example, +// +// import "google/protobuf/descriptor.proto"; +func (f *formatter) writeImport(importNode *ast.ImportNode, forceCompact bool) { + f.writeStartMaybeCompact(importNode.Keyword, forceCompact) + f.Space() + // We don't want to write the "public" and "weak" nodes + // if they aren't defined. One could be set, but never both. + switch { + case importNode.Public != nil: + f.writeInline(importNode.Public) + f.Space() + case importNode.Weak != nil: + f.writeInline(importNode.Weak) + f.Space() + } + f.writeInline(importNode.Name) + f.writeLineEnd(importNode.Semicolon) +} + +// writeFileOption writes a file option. This function is slightly +// different than f.writeOption because file options are sorted at +// the top of the file, and leading comments are adjusted accordingly. +func (f *formatter) writeFileOption(optionNode *ast.OptionNode, forceCompact bool) { + f.writeStartMaybeCompact(optionNode.Keyword, forceCompact) + f.Space() + f.writeNode(optionNode.Name) + f.Space() + f.writeInline(optionNode.Equals) + if node, ok := optionNode.Val.(*ast.CompoundStringLiteralNode); ok { + // Compound string literals are written across multiple lines + // immediately after the '=', so we don't need a trailing + // space in the option prefix. + f.writeCompoundStringLiteralIndentEndInline(node) + f.writeLineEnd(optionNode.Semicolon) + return + } + f.Space() + f.writeInline(optionNode.Val) + f.writeLineEnd(optionNode.Semicolon) +} + +// writeOption writes an option. +// +// For example, +// +// option go_package = "github.com/foo/bar"; +func (f *formatter) writeOption(optionNode *ast.OptionNode) { + f.writeOptionPrefix(optionNode) + if optionNode.Semicolon != nil { + if node, ok := optionNode.Val.(*ast.CompoundStringLiteralNode); ok { + // Compound string literals are written across multiple lines + // immediately after the '=', so we don't need a trailing + // space in the option prefix. + f.writeCompoundStringLiteralIndentEndInline(node) + f.writeLineEnd(optionNode.Semicolon) + return + } + f.writeInline(optionNode.Val) + f.writeLineEnd(optionNode.Semicolon) + return + } + + if node, ok := optionNode.Val.(*ast.CompoundStringLiteralNode); ok { + f.writeCompoundStringLiteralIndent(node) + return + } + f.writeInline(optionNode.Val) +} + +// writeLastCompactOption writes a compact option but preserves its the +// trailing end comments. This is only used for the last compact option +// since it's the only time a trailing ',' will be omitted. +// +// For example, +// +// [ +// deprecated = true, +// json_name = "something" // Trailing comment on the last element. +// ] +func (f *formatter) writeLastCompactOption(optionNode *ast.OptionNode) { + f.writeOptionPrefix(optionNode) + f.writeLineEnd(optionNode.Val) +} + +// writeOptionValue writes the option prefix, which makes up all of the +// option's definition, excluding the final token(s). +// +// For example, +// +// deprecated = +func (f *formatter) writeOptionPrefix(optionNode *ast.OptionNode) { + if optionNode.Keyword != nil { + // Compact options don't have the keyword. + f.writeStart(optionNode.Keyword) + f.Space() + f.writeNode(optionNode.Name) + } else { + f.writeStart(optionNode.Name) + } + f.Space() + f.writeInline(optionNode.Equals) + f.Space() +} + +// writeOptionName writes an option name. +// +// For example, +// +// go_package +// (custom.thing) +// (custom.thing).bridge.(another.thing) +func (f *formatter) writeOptionName(optionNameNode *ast.OptionNameNode) { + for i := 0; i < len(optionNameNode.Parts); i++ { + if f.inCompactOptions && i == 0 { + // The leading comments of the first token (either open rune or the + // name) will have already been written, so we need to handle this + // case specially. + fieldReferenceNode := optionNameNode.Parts[0] + if fieldReferenceNode.Open != nil { + f.writeNode(fieldReferenceNode.Open) + if info := f.nodeInfo(fieldReferenceNode.Open); info.TrailingComments().Len() > 0 { + f.writeInlineComments(info.TrailingComments()) + } + f.writeInline(fieldReferenceNode.Name) + } else { + f.writeNode(fieldReferenceNode.Name) + if info := f.nodeInfo(fieldReferenceNode.Name); info.TrailingComments().Len() > 0 { + f.writeInlineComments(info.TrailingComments()) + } + } + if fieldReferenceNode.Close != nil { + f.writeInline(fieldReferenceNode.Close) + } + continue + } + if i > 0 { + // The length of this slice must be exactly len(Parts)-1. + f.writeInline(optionNameNode.Dots[i-1]) + } + f.writeNode(optionNameNode.Parts[i]) + } +} + +// writeMessage writes the message node. +// +// For example, +// +// message Foo { +// option deprecated = true; +// reserved 50 to 100; +// extensions 150 to 200; +// +// message Bar { +// string name = 1; +// } +// enum Baz { +// BAZ_UNSPECIFIED = 0; +// } +// extend Bar { +// string value = 2; +// } +// +// Bar bar = 1; +// Baz baz = 2; +// } +func (f *formatter) writeMessage(messageNode *ast.MessageNode) { + var elementWriterFunc func() + if len(messageNode.Decls) != 0 { + elementWriterFunc = func() { + nodes := make([]ast.Node, len(messageNode.Decls)) + for i, n := range messageNode.Decls { + nodes[i] = n + } + f.writeNodes(nodes) + } + } + f.writeStart(messageNode.Keyword) + f.Space() + f.writeInline(messageNode.Name) + f.Space() + f.writeCompositeTypeBody( + messageNode.OpenBrace, + messageNode.CloseBrace, + elementWriterFunc, + ) +} + +// writeMessageLiteral writes a message literal. +// +// For example, +// +// { +// foo: 1 +// foo: 2 +// foo: 3 +// bar: < +// name:"abc" +// id:123 +// > +// } +func (f *formatter) writeMessageLiteral(messageLiteralNode *ast.MessageLiteralNode) { + if f.maybeWriteCompactMessageLiteral(messageLiteralNode, false) { + return + } + var elementWriterFunc func() + if len(messageLiteralNode.Elements) > 0 { + elementWriterFunc = func() { + f.writeMessageLiteralElements(messageLiteralNode) + } + } + f.writeCompositeValueBody( + messageLiteralOpen(messageLiteralNode), + messageLiteralClose(messageLiteralNode), + elementWriterFunc, + ) +} + +// writeMessageLiteral writes a message literal suitable for +// an element in an array literal. +func (f *formatter) writeMessageLiteralForArray( + messageLiteralNode *ast.MessageLiteralNode, + lastElement bool, +) { + if f.maybeWriteCompactMessageLiteral(messageLiteralNode, true) { + return + } + var elementWriterFunc func() + if len(messageLiteralNode.Elements) > 0 { + elementWriterFunc = func() { + f.writeMessageLiteralElements(messageLiteralNode) + } + } + closeWriter := f.writeBodyEndInline + if lastElement { + closeWriter = f.writeBodyEnd + } + f.writeBody( + messageLiteralOpen(messageLiteralNode), + messageLiteralClose(messageLiteralNode), + elementWriterFunc, + f.writeOpenBracePrefixForArray, + closeWriter, + ) +} + +func (f *formatter) maybeWriteCompactMessageLiteral( + messageLiteralNode *ast.MessageLiteralNode, + inArrayLiteral bool, +) bool { + if len(messageLiteralNode.Elements) == 0 || len(messageLiteralNode.Elements) > 1 || + f.hasInteriorComments(messageLiteralNode.Children()...) || + messageLiteralHasNestedMessageOrArray(messageLiteralNode) { + return false + } + // messages with a single scalar field and no comments can be + // printed all on one line + openNode := messageLiteralOpen(messageLiteralNode) + closeNode := messageLiteralClose(messageLiteralNode) + if inArrayLiteral { + f.Indent(openNode) + } + f.writeInline(openNode) + fieldNode := messageLiteralNode.Elements[0] + f.writeInline(fieldNode.Name) + if fieldNode.Sep != nil { + f.writeInline(fieldNode.Sep) + } else { + f.WriteString(":") + } + f.Space() + if messageLiteralNode.Seps[0] != nil { + // We are dropping the optional trailing separator. If it had + // trailing comments and the value does not, move the separator's + // trailing comment to the value. + sepTrailingComments := f.nodeInfo(messageLiteralNode.Seps[0]).TrailingComments() + if sepTrailingComments.Len() > 0 && + f.nodeInfo(fieldNode.Val).TrailingComments().Len() == 0 { + f.setTrailingComments(fieldNode.Val, sepTrailingComments) + } + } + f.writeInline(fieldNode.Val) + f.writeInline(closeNode) + return true +} + +func messageLiteralHasNestedMessageOrArray(messageLiteralNode *ast.MessageLiteralNode) bool { + for _, elem := range messageLiteralNode.Elements { + switch elem.Val.(type) { + case *ast.ArrayLiteralNode, *ast.MessageLiteralNode: + return true + } + } + return false +} + +func arrayLiteralHasNestedMessageOrArray(arrayLiteralNode *ast.ArrayLiteralNode) bool { + for _, elem := range arrayLiteralNode.Elements { + switch elem.(type) { + case *ast.ArrayLiteralNode, *ast.MessageLiteralNode: + return true + } + } + return false +} + +// writeMessageLiteralElements writes the message literal's elements. +// +// For example, +// +// foo: 1 +// foo: 2 +func (f *formatter) writeMessageLiteralElements(messageLiteralNode *ast.MessageLiteralNode) { + for i := 0; i < len(messageLiteralNode.Elements); i++ { + // Separators ("," or ";") are optional. To avoid inconsistent formatted output, + // we suppress them, since they aren't needed. So we just write the element and + // ignore any optional separator in the AST. + if messageLiteralNode.Seps[i] != nil { + // Since we are dropping the optional trailing separator, we should + // possibly move its trailing comment to the element value so we don't + // lose it. Skip this step if the value already has a trailing comment. + sepTrailingComments := f.nodeInfo(messageLiteralNode.Seps[i]).TrailingComments() + if sepTrailingComments.Len() > 0 && + f.nodeInfo(messageLiteralNode.Elements[i].Val).TrailingComments().Len() == 0 { + f.setTrailingComments(messageLiteralNode.Elements[i].Val, sepTrailingComments) + } + } + f.writeNode(messageLiteralNode.Elements[i]) + } +} + +// writeMessageField writes the message field node, and concludes the +// line without leaving room for a trailing separator in the parent +// message literal. +func (f *formatter) writeMessageField(messageFieldNode *ast.MessageFieldNode) { + f.writeMessageFieldPrefix(messageFieldNode) + if compoundStringLiteral, ok := messageFieldNode.Val.(*ast.CompoundStringLiteralNode); ok { + f.writeCompoundStringLiteralIndent(compoundStringLiteral) + return + } + f.writeLineEnd(messageFieldNode.Val) +} + +// writeMessageFieldPrefix writes the message field node as a single line. +// +// For example, +// +// foo:"bar" +func (f *formatter) writeMessageFieldPrefix(messageFieldNode *ast.MessageFieldNode) { + // The comments need to be written as a multiline comment above + // the message field name. + // + // Note that this is different than how field reference nodes are + // normally formatted in-line (i.e. as option name components). + fieldReferenceNode := messageFieldNode.Name + if fieldReferenceNode.Open != nil { + f.writeStart(fieldReferenceNode.Open) + if fieldReferenceNode.URLPrefix != nil { + f.writeInline(fieldReferenceNode.URLPrefix) + f.writeInline(fieldReferenceNode.Slash) + } + f.writeInline(fieldReferenceNode.Name) + } else { + f.writeStart(fieldReferenceNode.Name) + } + if fieldReferenceNode.Close != nil { + f.writeInline(fieldReferenceNode.Close) + } + // The colon separator is optional sometimes, but we don't have enough + // information here to know whether it's necessary. For more consistent + // output, just always include it. + if messageFieldNode.Sep != nil { + f.writeInline(messageFieldNode.Sep) + } else { + f.WriteString(":") + } + f.Space() +} + +// writeEnum writes the enum node. +// +// For example, +// +// enum Foo { +// option deprecated = true; +// reserved 1 to 5; +// +// FOO_UNSPECIFIED = 0; +// } +func (f *formatter) writeEnum(enumNode *ast.EnumNode) { + var elementWriterFunc func() + if len(enumNode.Decls) > 0 { + elementWriterFunc = func() { + nodes := make([]ast.Node, len(enumNode.Decls)) + for i, n := range enumNode.Decls { + nodes[i] = n + } + f.writeNodes(nodes) + } + } + f.writeStart(enumNode.Keyword) + f.Space() + f.writeInline(enumNode.Name) + f.Space() + f.writeCompositeTypeBody( + enumNode.OpenBrace, + enumNode.CloseBrace, + elementWriterFunc, + ) +} + +// writeEnumValue writes the enum value as a single line. If the enum has +// compact options, it will be written across multiple lines. +// +// For example, +// +// FOO_UNSPECIFIED = 1 [ +// deprecated = true +// ]; +func (f *formatter) writeEnumValue(enumValueNode *ast.EnumValueNode) { + f.writeStart(enumValueNode.Name) + f.Space() + f.writeInline(enumValueNode.Equals) + f.Space() + f.writeInline(enumValueNode.Number) + if enumValueNode.Options != nil { + f.Space() + f.writeNode(enumValueNode.Options) + } + f.writeLineEnd(enumValueNode.Semicolon) +} + +// writeField writes the field node as a single line. If the field has +// compact options, it will be written across multiple lines. +// +// For example, +// +// repeated string name = 1 [ +// deprecated = true, +// json_name = "name" +// ]; +func (f *formatter) writeField(fieldNode *ast.FieldNode) { + // We need to handle the comments for the field label specially since + // a label might not be defined, but it has the leading comments attached + // to it. + n := f.astNodeMapping[fieldNode] + fullType := f.fieldTypeMapping[n] + t := ast.NewIdentNode(fullType, fieldNode.FldType.Start()) + + if fieldNode.Label.KeywordNode != nil { + f.writeStart(fieldNode.Label) + f.Space() + //f.writeInline(fieldNode.FldType) + f.writeInline(t) + } else { + // If a label was not written, the multiline comments will be + // attached to the type. + if compoundIdentNode, ok := fieldNode.FldType.(*ast.CompoundIdentNode); ok { + f.writeCompountIdentForFieldName(compoundIdentNode) + } else { + //f.writeStart(fieldNode.FldType) + f.writeStart(t) + } + } + f.Space() + f.writeInline(fieldNode.Name) + f.Space() + f.writeInline(fieldNode.Equals) + f.Space() + f.writeInline(fieldNode.Tag) + if fieldNode.Options != nil { + f.Space() + f.writeNode(fieldNode.Options) + } + f.writeLineEnd(fieldNode.Semicolon) +} + +// writeMapField writes a map field (e.g. 'map pairs = 1;'). +func (f *formatter) writeMapField(mapFieldNode *ast.MapFieldNode) { + f.writeNode(mapFieldNode.MapType) + f.Space() + f.writeInline(mapFieldNode.Name) + f.Space() + f.writeInline(mapFieldNode.Equals) + f.Space() + f.writeInline(mapFieldNode.Tag) + if mapFieldNode.Options != nil { + f.Space() + f.writeNode(mapFieldNode.Options) + } + f.writeLineEnd(mapFieldNode.Semicolon) +} + +// writeMapType writes a map type (e.g. 'map'). +func (f *formatter) writeMapType(mapTypeNode *ast.MapTypeNode) { + f.writeStart(mapTypeNode.Keyword) + f.writeInline(mapTypeNode.OpenAngle) + f.writeInline(mapTypeNode.KeyType) + f.writeInline(mapTypeNode.Comma) + f.Space() + f.writeInline(mapTypeNode.ValueType) + f.writeInline(mapTypeNode.CloseAngle) +} + +// writeFieldReference writes a field reference (e.g. '(foo.bar)'). +func (f *formatter) writeFieldReference(fieldReferenceNode *ast.FieldReferenceNode) { + if fieldReferenceNode.Open != nil { + f.writeInline(fieldReferenceNode.Open) + } + f.writeInline(fieldReferenceNode.Name) + if fieldReferenceNode.Close != nil { + f.writeInline(fieldReferenceNode.Close) + } +} + +// writeExtend writes the extend node. +// +// For example, +// +// extend google.protobuf.FieldOptions { +// bool redacted = 33333; +// } +func (f *formatter) writeExtend(extendNode *ast.ExtendNode) { + var elementWriterFunc func() + if len(extendNode.Decls) > 0 { + elementWriterFunc = func() { + nodes := make([]ast.Node, len(extendNode.Decls)) + for i, n := range extendNode.Decls { + nodes[i] = n + } + f.writeNodes(nodes) + } + } + f.writeStart(extendNode.Keyword) + f.Space() + f.writeInline(extendNode.Extendee) + f.Space() + f.writeCompositeTypeBody( + extendNode.OpenBrace, + extendNode.CloseBrace, + elementWriterFunc, + ) +} + +// writeService writes the service node. +// +// For example, +// +// service FooService { +// option deprecated = true; +// +// rpc Foo(FooRequest) returns (FooResponse) {}; +func (f *formatter) writeService(serviceNode *ast.ServiceNode) { + var elementWriterFunc func() + if len(serviceNode.Decls) > 0 { + elementWriterFunc = func() { + nodes := make([]ast.Node, len(serviceNode.Decls)) + for i, n := range serviceNode.Decls { + nodes[i] = n + } + f.writeNodes(nodes) + } + } + f.writeStart(serviceNode.Keyword) + f.Space() + f.writeInline(serviceNode.Name) + f.Space() + f.writeCompositeTypeBody( + serviceNode.OpenBrace, + serviceNode.CloseBrace, + elementWriterFunc, + ) +} + +// writeRPC writes the RPC node. RPCs are formatted in +// the following order: +// +// For example, +// +// rpc Foo(FooRequest) returns (FooResponse) { +// option deprecated = true; +// }; +func (f *formatter) writeRPC(rpcNode *ast.RPCNode) { + var elementWriterFunc func() + if len(rpcNode.Decls) > 0 { + elementWriterFunc = func() { + nodes := make([]ast.Node, len(rpcNode.Decls)) + for i, n := range rpcNode.Decls { + nodes[i] = n + } + f.writeNodes(nodes) + } + } + f.writeStart(rpcNode.Keyword) + f.Space() + f.writeInline(rpcNode.Name) + f.writeInline(rpcNode.Input) + f.Space() + f.writeInline(rpcNode.Returns) + f.Space() + f.writeInline(rpcNode.Output) + if rpcNode.OpenBrace == nil { + // This RPC doesn't have any elements, so we prefer the + // ';' form. + // + // rpc Ping(PingRequest) returns (PingResponse); + // + f.writeLineEnd(rpcNode.Semicolon) + return + } + f.Space() + f.writeCompositeTypeBody( + rpcNode.OpenBrace, + rpcNode.CloseBrace, + elementWriterFunc, + ) +} + +// writeRPCType writes the RPC type node (e.g. (stream foo.Bar)). +func (f *formatter) writeRPCType(rpcTypeNode *ast.RPCTypeNode) { + f.writeInline(rpcTypeNode.OpenParen) + if rpcTypeNode.Stream != nil { + f.writeInline(rpcTypeNode.Stream) + f.Space() + } + f.writeInline(rpcTypeNode.MessageType) + f.writeInline(rpcTypeNode.CloseParen) +} + +// writeOneOf writes the oneof node. +// +// For example, +// +// oneof foo { +// option deprecated = true; +// +// string name = 1; +// int number = 2; +// } +func (f *formatter) writeOneOf(oneOfNode *ast.OneofNode) { + var elementWriterFunc func() + if len(oneOfNode.Decls) > 0 { + elementWriterFunc = func() { + nodes := make([]ast.Node, len(oneOfNode.Decls)) + for i, n := range oneOfNode.Decls { + nodes[i] = n + } + f.writeNodes(nodes) + } + } + f.writeStart(oneOfNode.Keyword) + f.Space() + f.writeInline(oneOfNode.Name) + f.Space() + f.writeCompositeTypeBody( + oneOfNode.OpenBrace, + oneOfNode.CloseBrace, + elementWriterFunc, + ) +} + +// writeGroup writes the group node. +// +// For example, +// +// optional group Key = 4 [ +// deprecated = true, +// json_name = "key" +// ] { +// optional uint64 id = 1; +// optional string name = 2; +// } +func (f *formatter) writeGroup(groupNode *ast.GroupNode) { + var elementWriterFunc func() + if len(groupNode.Decls) > 0 { + elementWriterFunc = func() { + nodes := make([]ast.Node, len(groupNode.Decls)) + for i, n := range groupNode.Decls { + nodes[i] = n + } + f.writeNodes(nodes) + } + } + // We need to handle the comments for the group label specially since + // a label might not be defined, but it has the leading comments attached + // to it. + if groupNode.Label.KeywordNode != nil { + f.writeStart(groupNode.Label) + f.Space() + f.writeInline(groupNode.Keyword) + } else { + // If a label was not written, the multiline comments will be + // attached to the keyword. + f.writeStart(groupNode.Keyword) + } + f.Space() + f.writeInline(groupNode.Name) + f.Space() + f.writeInline(groupNode.Equals) + f.Space() + f.writeInline(groupNode.Tag) + if groupNode.Options != nil { + f.Space() + f.writeNode(groupNode.Options) + } + f.Space() + f.writeCompositeTypeBody( + groupNode.OpenBrace, + groupNode.CloseBrace, + elementWriterFunc, + ) +} + +// writeExtensionRange writes the extension range node. +// +// For example, +// +// extensions 5-10, 100 to max [ +// deprecated = true +// ]; +func (f *formatter) writeExtensionRange(extensionRangeNode *ast.ExtensionRangeNode) { + f.writeStart(extensionRangeNode.Keyword) + f.Space() + for i := 0; i < len(extensionRangeNode.Ranges); i++ { + if i > 0 { + // The length of this slice must be exactly len(Ranges)-1. + f.writeInline(extensionRangeNode.Commas[i-1]) + f.Space() + } + f.writeNode(extensionRangeNode.Ranges[i]) + } + if extensionRangeNode.Options != nil { + f.Space() + f.writeNode(extensionRangeNode.Options) + } + f.writeLineEnd(extensionRangeNode.Semicolon) +} + +// writeReserved writes a reserved node. +// +// For example, +// +// reserved 5-10, 100 to max; +func (f *formatter) writeReserved(reservedNode *ast.ReservedNode) { + f.writeStart(reservedNode.Keyword) + // Either names or ranges will be set, but never both. + elements := make([]ast.Node, 0, len(reservedNode.Names)+len(reservedNode.Ranges)) + switch { + case reservedNode.Names != nil: + for _, nameNode := range reservedNode.Names { + elements = append(elements, nameNode) + } + case reservedNode.Identifiers != nil: + for _, identNode := range reservedNode.Identifiers { + elements = append(elements, identNode) + } + case reservedNode.Ranges != nil: + for _, rangeNode := range reservedNode.Ranges { + elements = append(elements, rangeNode) + } + } + f.Space() + for i := 0; i < len(elements); i++ { + if i > 0 { + // The length of this slice must be exactly len({Names,Ranges})-1. + f.writeInline(reservedNode.Commas[i-1]) + f.Space() + } + f.writeInline(elements[i]) + } + f.writeLineEnd(reservedNode.Semicolon) +} + +// writeRange writes the given range node (e.g. '1 to max'). +func (f *formatter) writeRange(rangeNode *ast.RangeNode) { + f.writeInline(rangeNode.StartVal) + if rangeNode.To != nil { + f.Space() + f.writeInline(rangeNode.To) + } + // Either EndVal or Max will be set, but never both. + switch { + case rangeNode.EndVal != nil: + f.Space() + f.writeInline(rangeNode.EndVal) + case rangeNode.Max != nil: + f.Space() + f.writeInline(rangeNode.Max) + } +} + +// writeCompactOptions writes a compact options node. +// +// For example, +// +// [ +// deprecated = true, +// json_name = "something" +// ] +func (f *formatter) writeCompactOptions(compactOptionsNode *ast.CompactOptionsNode) { + f.inCompactOptions = true + defer func() { + f.inCompactOptions = false + }() + if len(compactOptionsNode.Options) == 1 && + !f.hasInteriorComments(compactOptionsNode.OpenBracket, compactOptionsNode.Options[0].Name) { + // If there's only a single compact scalar option without comments, we can write it + // in-line. For example: + // + // [deprecated = true] + // + // However, this does not include the case when the '[' has trailing comments, + // or the option name has leading comments. In those cases, we write the option + // across multiple lines. For example: + // + // [ + // // This type is deprecated. + // deprecated = true + // ] + // + optionNode := compactOptionsNode.Options[0] + f.writeInline(compactOptionsNode.OpenBracket) + f.writeInline(optionNode.Name) + f.Space() + f.writeInline(optionNode.Equals) + if node, ok := optionNode.Val.(*ast.CompoundStringLiteralNode); ok { + // If there's only a single compact option, the value needs to + // write its comments (if any) in a way that preserves the closing ']'. + f.writeCompoundStringLiteralNoIndentEndInline(node) + f.writeInline(compactOptionsNode.CloseBracket) + return + } + f.Space() + f.writeInline(optionNode.Val) + f.writeInline(compactOptionsNode.CloseBracket) + return + } + var elementWriterFunc func() + if len(compactOptionsNode.Options) > 0 { + elementWriterFunc = func() { + sort.Slice(compactOptionsNode.Options, func(i, j int) bool { + // The default options (e.g. cc_enable_arenas) should always + // be sorted above custom options (which are identified by a + // leading '('). + left := stringForOptionName(compactOptionsNode.Options[i].Name) + right := stringForOptionName(compactOptionsNode.Options[j].Name) + if strings.HasPrefix(left, "(") && !strings.HasPrefix(right, "(") { + // Prefer the default option on the right. + return false + } + if !strings.HasPrefix(left, "(") && strings.HasPrefix(right, "(") { + // Prefer the default option on the left. + return true + } + // Both options are custom, so we defer to the standard sorting. + return left < right + }) + for i, opt := range compactOptionsNode.Options { + if i == len(compactOptionsNode.Options)-1 { + // The last element won't have a trailing comma. + f.writeLastCompactOption(opt) + return + } + f.writeNode(opt) + f.writeLineEnd(compactOptionsNode.Commas[i]) + } + } + } + f.writeCompositeValueBody( + compactOptionsNode.OpenBracket, + compactOptionsNode.CloseBracket, + elementWriterFunc, + ) +} + +func (f *formatter) hasInteriorComments(nodes ...ast.Node) bool { + for i, n := range nodes { + // interior comments mean we ignore leading comments on first + // token and trailing comments on the last one + info := f.nodeInfo(n) + if i > 0 && info.LeadingComments().Len() > 0 { + return true + } + if i < len(nodes)-1 && info.TrailingComments().Len() > 0 { + return true + } + } + return false +} + +// writeArrayLiteral writes an array literal across multiple lines. +// +// For example, +// +// [ +// "foo", +// "bar" +// ] +func (f *formatter) writeArrayLiteral(arrayLiteralNode *ast.ArrayLiteralNode) { + if len(arrayLiteralNode.Elements) == 1 && + !f.hasInteriorComments(arrayLiteralNode.Children()...) && + !arrayLiteralHasNestedMessageOrArray(arrayLiteralNode) { + // arrays with a single scalar value and no comments can be + // printed all on one line + valueNode := arrayLiteralNode.Elements[0] + f.writeInline(arrayLiteralNode.OpenBracket) + f.writeInline(valueNode) + f.writeInline(arrayLiteralNode.CloseBracket) + return + } + + var elementWriterFunc func() + if len(arrayLiteralNode.Elements) > 0 { + elementWriterFunc = func() { + for i := 0; i < len(arrayLiteralNode.Elements); i++ { + lastElement := i == len(arrayLiteralNode.Elements)-1 + if compositeNode, ok := arrayLiteralNode.Elements[i].(ast.CompositeNode); ok { + f.writeCompositeValueForArrayLiteral(compositeNode, lastElement) + if !lastElement { + f.writeLineEnd(arrayLiteralNode.Commas[i]) + } + continue + } + if lastElement { + // The last element won't have a trailing comma. + f.writeLineElement(arrayLiteralNode.Elements[i]) + return + } + f.writeStart(arrayLiteralNode.Elements[i]) + f.writeLineEnd(arrayLiteralNode.Commas[i]) + } + } + } + f.writeCompositeValueBody( + arrayLiteralNode.OpenBracket, + arrayLiteralNode.CloseBracket, + elementWriterFunc, + ) +} + +// writeCompositeForArrayLiteral writes the composite node in a way that's suitable +// for array literals. In general, signed integers and compound strings should have their +// comments written in-line because they are one of many components in a single line. +// +// However, each of these composite types occupy a single line in an array literal, +// so they need their comments to be formatted like a standalone node. +// +// For example, +// +// option (value) = /* In-line comment for '-42' */ -42; +// +// option (thing) = { +// values: [ +// // Leading comment on -42. +// -42, // Trailing comment on -42. +// ] +// } +// +// The lastElement boolean is used to signal whether or not the composite value +// should be written as the last element (i.e. it doesn't have a trailing comma). +func (f *formatter) writeCompositeValueForArrayLiteral( + compositeNode ast.CompositeNode, + lastElement bool, +) { + switch node := compositeNode.(type) { + case *ast.CompoundStringLiteralNode: + f.writeCompoundStringLiteralForArray(node, lastElement) + case *ast.NegativeIntLiteralNode: + f.writeNegativeIntLiteralForArray(node, lastElement) + case *ast.SignedFloatLiteralNode: + f.writeSignedFloatLiteralForArray(node, lastElement) + case *ast.MessageLiteralNode: + f.writeMessageLiteralForArray(node, lastElement) + default: + f.err = multierr.Append(f.err, fmt.Errorf("unexpected array value node %T", node)) + } +} + +// writeCompositeTypeBody writes the body of a composite type, e.g. message, enum, extend, oneof, etc. +func (f *formatter) writeCompositeTypeBody( + openBrace *ast.RuneNode, + closeBrace *ast.RuneNode, + elementWriterFunc func(), +) { + f.writeBody( + openBrace, + closeBrace, + elementWriterFunc, + f.writeOpenBracePrefix, + f.writeBodyEnd, + ) +} + +// writeCompositeValueBody writes the body of a composite value, e.g. compact options, +// array literal, etc. We need to handle the ']' different than composite types because +// there could be more tokens following the final ']'. +func (f *formatter) writeCompositeValueBody( + openBrace *ast.RuneNode, + closeBrace *ast.RuneNode, + elementWriterFunc func(), +) { + f.writeBody( + openBrace, + closeBrace, + elementWriterFunc, + f.writeOpenBracePrefix, + f.writeBodyEndInline, + ) +} + +// writeBody writes the body of a type or value, e.g. message, enum, compact options, etc. +// The elementWriterFunc is used to write the declarations within the composite type (e.g. +// fields in a message). The openBraceWriterFunc and closeBraceWriterFunc functions are used +// to customize how the '{' and '} nodes are written, respectively. +func (f *formatter) writeBody( + openBrace *ast.RuneNode, + closeBrace *ast.RuneNode, + elementWriterFunc func(), + openBraceWriterFunc func(ast.Node), + closeBraceWriterFunc func(ast.Node, bool), +) { + if elementWriterFunc == nil && !f.hasInteriorComments(openBrace, closeBrace) { + // completely empty body + f.writeInline(openBrace) + closeBraceWriterFunc(closeBrace, true) + return + } + + openBraceWriterFunc(openBrace) + if elementWriterFunc != nil { + elementWriterFunc() + } + closeBraceWriterFunc(closeBrace, false) +} + +// writeOpenBracePrefix writes the open brace with its leading comments in-line. +// This is used for nearly every use case of f.writeBody, excluding the instances +// in array literals. +func (f *formatter) writeOpenBracePrefix(openBrace ast.Node) { + defer f.SetPreviousNode(openBrace) + info := f.nodeInfo(openBrace) + if info.LeadingComments().Len() > 0 { + f.writeInlineComments(info.LeadingComments()) + if info.LeadingWhitespace() != "" { + f.Space() + } + } + f.writeNode(openBrace) + if info.TrailingComments().Len() > 0 { + f.writeTrailingEndComments(info.TrailingComments()) + } else { + f.P("") + } +} + +// writeOpenBracePrefixForArray writes the open brace with its leading comments +// on multiple lines. This is only used for message literals in arrays. +func (f *formatter) writeOpenBracePrefixForArray(openBrace ast.Node) { + defer f.SetPreviousNode(openBrace) + info := f.nodeInfo(openBrace) + if info.LeadingComments().Len() > 0 { + f.writeMultilineComments(info.LeadingComments()) + } + f.Indent(openBrace) + f.writeNode(openBrace) + if info.TrailingComments().Len() > 0 { + f.writeTrailingEndComments(info.TrailingComments()) + } else { + f.P("") + } +} + +// writeCompoundIdent writes a compound identifier (e.g. '.com.foo.Bar'). +func (f *formatter) writeCompoundIdent(compoundIdentNode *ast.CompoundIdentNode) { + if compoundIdentNode.LeadingDot != nil { + f.writeInline(compoundIdentNode.LeadingDot) + } + for i := 0; i < len(compoundIdentNode.Components); i++ { + if i > 0 { + // The length of this slice must be exactly len(Components)-1. + f.writeInline(compoundIdentNode.Dots[i-1]) + } + f.writeInline(compoundIdentNode.Components[i]) + } +} + +// writeCompountIdentForFieldName writes a compound identifier, but handles comments +// specially for field names. +// +// For example, +// +// message Foo { +// // These are comments attached to bar. +// bar.v1.Bar bar = 1; +// } +func (f *formatter) writeCompountIdentForFieldName(compoundIdentNode *ast.CompoundIdentNode) { + if compoundIdentNode.LeadingDot != nil { + f.writeStart(compoundIdentNode.LeadingDot) + } + for i := 0; i < len(compoundIdentNode.Components); i++ { + if i == 0 && compoundIdentNode.LeadingDot == nil { + f.writeStart(compoundIdentNode.Components[i]) + continue + } + if i > 0 { + // The length of this slice must be exactly len(Components)-1. + f.writeInline(compoundIdentNode.Dots[i-1]) + } + f.writeInline(compoundIdentNode.Components[i]) + } +} + +// writeFieldLabel writes the field label node. +// +// For example, +// +// optional +// repeated +// required +func (f *formatter) writeFieldLabel(fieldLabel ast.FieldLabel) { + f.WriteString(fieldLabel.Val) +} + +// writeCompoundStringLiteral writes a compound string literal value. +// +// For example, +// +// "one," +// "two," +// "three" +func (f *formatter) writeCompoundStringLiteral( + compoundStringLiteralNode *ast.CompoundStringLiteralNode, + needsIndent bool, + hasTrailingPunctuation bool, +) { + f.P("") + if needsIndent { + f.In() + } + for i, child := range compoundStringLiteralNode.Children() { + if hasTrailingPunctuation && i == len(compoundStringLiteralNode.Children())-1 { + // inline because there may be a subsequent comma or punctuation from enclosing element + f.writeStart(child) + break + } + f.writeLineElement(child) + } + if needsIndent { + f.Out() + } +} + +func (f *formatter) writeCompoundStringLiteralIndent( + compoundStringLiteralNode *ast.CompoundStringLiteralNode, +) { + f.writeCompoundStringLiteral(compoundStringLiteralNode, true, false) +} + +func (f *formatter) writeCompoundStringLiteralIndentEndInline( + compoundStringLiteralNode *ast.CompoundStringLiteralNode, +) { + f.writeCompoundStringLiteral(compoundStringLiteralNode, true, true) +} + +func (f *formatter) writeCompoundStringLiteralNoIndentEndInline( + compoundStringLiteralNode *ast.CompoundStringLiteralNode, +) { + f.writeCompoundStringLiteral(compoundStringLiteralNode, false, true) +} + +// writeCompoundStringLiteralForArray writes a compound string literal value, +// but writes its comments suitable for an element in an array literal. +// +// The lastElement boolean is used to signal whether or not the value should +// be written as the last element (i.e. it doesn't have a trailing comma). +func (f *formatter) writeCompoundStringLiteralForArray( + compoundStringLiteralNode *ast.CompoundStringLiteralNode, + lastElement bool, +) { + for i, child := range compoundStringLiteralNode.Children() { + if !lastElement && i == len(compoundStringLiteralNode.Children())-1 { + f.writeStart(child) + return + } + f.writeLineElement(child) + } +} + +// writeFloatLiteral writes a float literal value (e.g. '42.2'). +func (f *formatter) writeFloatLiteral(floatLiteralNode *ast.FloatLiteralNode) { + f.writeRaw(floatLiteralNode) +} + +// writeSignedFloatLiteral writes a signed float literal value (e.g. '-42.2'). +func (f *formatter) writeSignedFloatLiteral(signedFloatLiteralNode *ast.SignedFloatLiteralNode) { + f.writeInline(signedFloatLiteralNode.Sign) + f.writeInline(signedFloatLiteralNode.Float) +} + +// writeSignedFloatLiteralForArray writes a signed float literal value, but writes +// its comments suitable for an element in an array literal. +// +// The lastElement boolean is used to signal whether or not the value should +// be written as the last element (i.e. it doesn't have a trailing comma). +func (f *formatter) writeSignedFloatLiteralForArray( + signedFloatLiteralNode *ast.SignedFloatLiteralNode, + lastElement bool, +) { + f.writeStart(signedFloatLiteralNode.Sign) + if lastElement { + f.writeLineEnd(signedFloatLiteralNode.Float) + return + } + f.writeInline(signedFloatLiteralNode.Float) +} + +// writeSpecialFloatLiteral writes a special float literal value (e.g. "nan" or "inf"). +func (f *formatter) writeSpecialFloatLiteral(specialFloatLiteralNode *ast.SpecialFloatLiteralNode) { + f.WriteString(specialFloatLiteralNode.KeywordNode.Val) +} + +// writeStringLiteral writes a string literal value (e.g. "foo"). +// Note that the raw string is written as-is so that it preserves +// the quote style used in the original source. +func (f *formatter) writeStringLiteral(stringLiteralNode *ast.StringLiteralNode) { + f.writeRaw(stringLiteralNode) +} + +// writeUintLiteral writes a uint literal (e.g. '42'). +func (f *formatter) writeUintLiteral(uintLiteralNode *ast.UintLiteralNode) { + f.writeRaw(uintLiteralNode) +} + +// writeNegativeIntLiteral writes a negative int literal (e.g. '-42'). +func (f *formatter) writeNegativeIntLiteral(negativeIntLiteralNode *ast.NegativeIntLiteralNode) { + f.writeInline(negativeIntLiteralNode.Minus) + f.writeInline(negativeIntLiteralNode.Uint) +} + +func (f *formatter) writeRaw(n ast.Node) { + info := f.nodeInfo(n) + f.WriteString(info.RawText()) +} + +// writeNegativeIntLiteralForArray writes a negative int literal value, but writes +// its comments suitable for an element in an array literal. +// +// The lastElement boolean is used to signal whether or not the value should +// be written as the last element (i.e. it doesn't have a trailing comma). +func (f *formatter) writeNegativeIntLiteralForArray( + negativeIntLiteralNode *ast.NegativeIntLiteralNode, + lastElement bool, +) { + f.writeStart(negativeIntLiteralNode.Minus) + if lastElement { + f.writeLineEnd(negativeIntLiteralNode.Uint) + return + } + f.writeInline(negativeIntLiteralNode.Uint) +} + +// writeIdent writes an identifier (e.g. 'foo'). +func (f *formatter) writeIdent(identNode *ast.IdentNode) { + f.WriteString(identNode.Val) +} + +// writeKeyword writes a keyword (e.g. 'syntax'). +func (f *formatter) writeKeyword(keywordNode *ast.KeywordNode) { + f.WriteString(keywordNode.Val) +} + +// writeRune writes a rune (e.g. '='). +func (f *formatter) writeRune(runeNode *ast.RuneNode) { + if strings.ContainsRune("{[(<", runeNode.Rune) { + f.pendingIndent++ + } else if strings.ContainsRune("}])>", runeNode.Rune) { + f.pendingIndent-- + } + f.WriteString(string(runeNode.Rune)) +} + +// writeNodes writes nodes with sorted options. +func (f *formatter) writeNodes(nodes []ast.Node) { + optionNodes := []*ast.OptionNode{} + for _, node := range nodes { + if option, ok := node.(*ast.OptionNode); ok { + optionNodes = append(optionNodes, option) + } + } + + sort.Slice(optionNodes, func(i, j int) bool { + // The default options (e.g. cc_enable_arenas) should always + // be sorted above custom options (which are identified by a + // leading '('). + left := stringForOptionName(optionNodes[i].Name) + right := stringForOptionName(optionNodes[j].Name) + if strings.HasPrefix(left, "(") && !strings.HasPrefix(right, "(") { + // Prefer the default option on the right. + return false + } + if !strings.HasPrefix(left, "(") && strings.HasPrefix(right, "(") { + // Prefer the default option on the left. + return true + } + // Both options are custom, so we defer to the standard sorting. + return left < right + }) + + for _, node := range optionNodes { + f.writeNode(node) + } + + for _, node := range nodes { + if _, ok := node.(*ast.OptionNode); !ok { + f.writeNode(node) + } + } +} + +// writeNode writes the node by dispatching to a function tailored to its concrete type. +// +// Comments are handled in each respective write function so that it can determine whether +// to write the comments in-line or not. +func (f *formatter) writeNode(node ast.Node) { + switch element := node.(type) { + case *ast.ArrayLiteralNode: + f.writeArrayLiteral(element) + case *ast.CompactOptionsNode: + f.writeCompactOptions(element) + case *ast.CompoundIdentNode: + f.writeCompoundIdent(element) + case *ast.CompoundStringLiteralNode: + f.writeCompoundStringLiteralIndent(element) + case *ast.EnumNode: + f.writeEnum(element) + case *ast.EnumValueNode: + f.writeEnumValue(element) + case *ast.ExtendNode: + f.writeExtend(element) + case *ast.ExtensionRangeNode: + f.writeExtensionRange(element) + case ast.FieldLabel: + f.writeFieldLabel(element) + case *ast.FieldNode: + f.writeField(element) + case *ast.FieldReferenceNode: + f.writeFieldReference(element) + case *ast.FloatLiteralNode: + f.writeFloatLiteral(element) + case *ast.GroupNode: + f.writeGroup(element) + case *ast.IdentNode: + f.writeIdent(element) + case *ast.ImportNode: + f.writeImport(element, false) + case *ast.KeywordNode: + f.writeKeyword(element) + case *ast.MapFieldNode: + f.writeMapField(element) + case *ast.MapTypeNode: + f.writeMapType(element) + case *ast.MessageNode: + f.writeMessage(element) + case *ast.MessageFieldNode: + f.writeMessageField(element) + case *ast.MessageLiteralNode: + f.writeMessageLiteral(element) + case *ast.NegativeIntLiteralNode: + f.writeNegativeIntLiteral(element) + case *ast.OneofNode: + f.writeOneOf(element) + case *ast.OptionNode: + f.writeOption(element) + case *ast.OptionNameNode: + f.writeOptionName(element) + case *ast.PackageNode: + f.writePackage(element) + case *ast.RangeNode: + f.writeRange(element) + case *ast.ReservedNode: + f.writeReserved(element) + case *ast.RPCNode: + f.writeRPC(element) + case *ast.RPCTypeNode: + f.writeRPCType(element) + case *ast.RuneNode: + f.writeRune(element) + case *ast.ServiceNode: + f.writeService(element) + case *ast.SignedFloatLiteralNode: + f.writeSignedFloatLiteral(element) + case *ast.SpecialFloatLiteralNode: + f.writeSpecialFloatLiteral(element) + case *ast.StringLiteralNode: + f.writeStringLiteral(element) + case *ast.SyntaxNode: + f.writeSyntax(element) + case *ast.UintLiteralNode: + f.writeUintLiteral(element) + case *ast.EmptyDeclNode: + // Nothing to do here. + default: + f.err = multierr.Append(f.err, fmt.Errorf("unexpected node: %T", node)) + } +} + +// writeStart writes the node across as the start of a line. +// Start nodes have their leading comments written across +// multiple lines, but their trailing comments must be written +// in-line to preserve the line structure. +// +// For example, +// +// // Leading comment on 'message'. +// // Spread across multiple lines. +// message /* This is a trailing comment on 'message' */ Foo {} +// +// Newlines are preserved, so that any logical grouping of elements +// is maintained in the formatted result. +// +// For example, +// +// // Type represents a set of different types. +// enum Type { +// // Unspecified is the naming convention for default enum values. +// TYPE_UNSPECIFIED = 0; +// +// // The following elements are the real values. +// TYPE_ONE = 1; +// TYPE_TWO = 2; +// } +// +// Start nodes are always indented according to the formatter's +// current level of indentation (e.g. nested messages, fields, etc). +// +// Note that this is one of the most complex component of the formatter - it +// controls how each node should be separated from one another and preserves +// newlines in the original source. +func (f *formatter) writeStart(node ast.Node) { + f.writeStartMaybeCompact(node, true) +} + +func (f *formatter) writeStartMaybeCompact(node ast.Node, forceCompact bool) { + defer f.SetPreviousNode(node) + info := f.nodeInfo(node) + var ( + nodeNewlineCount = newlineCount(info.LeadingWhitespace()) + compact = forceCompact || isOpenBrace(f.previousNode) + ) + if length := info.LeadingComments().Len(); length > 0 { + // If leading comments are defined, the whitespace we care about + // is attached to the first comment. + f.writeMultilineCommentsMaybeCompact(info.LeadingComments(), forceCompact) + if !forceCompact && nodeNewlineCount > 1 { + // At this point, we're looking at the lines between + // a comment and the node its attached to. + // + // If the last comment is a standard comment, a single newline + // character is sufficient to warrant a separation of the + // two. + // + // If the last comment is a C-style comment, multiple newline + // characters are required because C-style comments don't consume + // a newline. + f.P("") + } + } else if !compact && nodeNewlineCount > 1 { + // If the previous node is an open brace, this is the first element + // in the body of a composite type, so we don't want to write a + // newline. This makes it so that trailing newlines are removed. + // + // For example, + // + // message Foo { + // + // string bar = 1; + // } + // + // Is formatted into the following: + // + // message Foo { + // string bar = 1; + // } + f.P("") + } + f.Indent(node) + f.writeNode(node) + if info.TrailingComments().Len() > 0 { + f.writeInlineComments(info.TrailingComments()) + } +} + +// writeInline writes the node and its surrounding comments in-line. +// +// This is useful for writing individual nodes like keywords, runes, +// string literals, etc. +// +// For example, +// +// // This is a leading comment on the syntax keyword. +// syntax = /* This is a leading comment on 'proto3' */" proto3"; +func (f *formatter) writeInline(node ast.Node) { + f.inline = true + defer func() { + f.inline = false + }() + if _, ok := node.(ast.CompositeNode); ok { + // We only want to write comments for terminal nodes. + // Otherwise comments accessible from CompositeNodes + // will be written twice. + f.writeNode(node) + return + } + defer f.SetPreviousNode(node) + info := f.nodeInfo(node) + if info.LeadingComments().Len() > 0 { + f.writeInlineComments(info.LeadingComments()) + if info.LeadingWhitespace() != "" { + f.Space() + } + } + f.writeNode(node) + f.writeInlineComments(info.TrailingComments()) +} + +// writeBodyEnd writes the node as the end of a body. +// Leading comments are written above the token across +// multiple lines, whereas the trailing comments are +// written in-line and preserve their format. +// +// Body end nodes are always indented according to the +// formatter's current level of indentation (e.g. nested +// messages). +// +// This is useful for writing a node that concludes a +// composite node: ']', '}', '>', etc. +// +// For example, +// +// message Foo { +// string bar = 1; +// // Leading comment on '}'. +// } // Trailing comment on '}. +func (f *formatter) writeBodyEnd(node ast.Node, leadingEndline bool) { + if _, ok := node.(ast.CompositeNode); ok { + // We only want to write comments for terminal nodes. + // Otherwise comments accessible from CompositeNodes + // will be written twice. + f.writeNode(node) + if f.lastWritten != '\n' { + f.P("") + } + return + } + defer f.SetPreviousNode(node) + info := f.nodeInfo(node) + if leadingEndline { + if info.LeadingComments().Len() > 0 { + f.writeInlineComments(info.LeadingComments()) + if info.LeadingWhitespace() != "" { + f.Space() + } + } + } else { + f.writeMultilineComments(info.LeadingComments()) + f.Indent(node) + } + f.writeNode(node) + f.writeTrailingEndComments(info.TrailingComments()) +} + +func (f *formatter) writeLineElement(node ast.Node) { + f.writeBodyEnd(node, false) +} + +// writeBodyEndInline writes the node as the end of a body. +// Leading comments are written above the token across +// multiple lines, whereas the trailing comments are +// written in-line and adapt their comment style if they +// exist. +// +// Body end nodes are always indented according to the +// formatter's current level of indentation (e.g. nested +// messages). +// +// This is useful for writing a node that concludes either +// compact options or an array literal. +// +// This is behaviorally similar to f.writeStart, but it ignores +// the preceding newline logic because these body ends should +// always be compact. +// +// For example, +// +// message Foo { +// string bar = 1 [ +// deprecated = true +// +// // Leading comment on ']'. +// ] /* Trailing comment on ']' */ ; +// } +func (f *formatter) writeBodyEndInline(node ast.Node, leadingInline bool) { + if _, ok := node.(ast.CompositeNode); ok { + // We only want to write comments for terminal nodes. + // Otherwise comments accessible from CompositeNodes + // will be written twice. + f.writeNode(node) + return + } + defer f.SetPreviousNode(node) + info := f.nodeInfo(node) + if leadingInline { + if info.LeadingComments().Len() > 0 { + f.writeInlineComments(info.LeadingComments()) + if info.LeadingWhitespace() != "" { + f.Space() + } + } + } else { + f.writeMultilineComments(info.LeadingComments()) + f.Indent(node) + } + f.writeNode(node) + if info.TrailingComments().Len() > 0 { + f.writeInlineComments(info.TrailingComments()) + } +} + +// writeLineEnd writes the node so that it ends a line. +// +// This is useful for writing individual nodes like ';' and other +// tokens that conclude the end of a single line. In this case, we +// don't want to transform the trailing comment's from '//' to C-style +// because it's not necessary. +// +// For example, +// +// // This is a leading comment on the syntax keyword. +// syntax = " proto3" /* This is a leading comment on the ';'; // This is a trailing comment on the ';'. +func (f *formatter) writeLineEnd(node ast.Node) { + if _, ok := node.(ast.CompositeNode); ok { + // We only want to write comments for terminal nodes. + // Otherwise comments accessible from CompositeNodes + // will be written twice. + f.writeNode(node) + if f.lastWritten != '\n' { + f.P("") + } + return + } + defer f.SetPreviousNode(node) + info := f.nodeInfo(node) + if info.LeadingComments().Len() > 0 { + f.writeInlineComments(info.LeadingComments()) + if info.LeadingWhitespace() != "" { + f.Space() + } + } + f.writeNode(node) + f.Space() + f.writeTrailingEndComments(info.TrailingComments()) +} + +// writeMultilineComments writes the given comments as a newline-delimited block. +// This is useful for both the beginning of a type (e.g. message, field, etc), as +// well as the trailing comments attached to the beginning of a body block (e.g. +// '{', '[', '<', etc). +// +// For example, +// +// // This is a comment spread across +// // multiple lines. +// message Foo {} +func (f *formatter) writeMultilineComments(comments ast.Comments) { + f.writeMultilineCommentsMaybeCompact(comments, false) +} + +func (f *formatter) writeMultilineCommentsMaybeCompact(comments ast.Comments, forceCompact bool) { + compact := forceCompact || isOpenBrace(f.previousNode) + for i := 0; i < comments.Len(); i++ { + comment := comments.Index(i) + if !compact && newlineCount(comment.LeadingWhitespace()) > 1 { + // Newlines between blocks of comments should be preserved. + // + // For example, + // + // // This is a license header + // // spread across multiple lines. + // + // // Package pet.v1 defines a PetStore API. + // package pet.v1; + // + f.P("") + } + compact = false + f.writeComment(comment.RawText()) + f.WriteString("\n") + } +} + +// writeInlineComments writes the given comments in-line. Standard comments are +// transformed to C-style comments so that we can safely write the comment in-line. +// +// Nearly all of these comments will already be C-style comments. The only cases we're +// preventing are when the type is defined across multiple lines. +// +// For example, given the following: +// +// extend . google. // in-line comment +// protobuf . +// ExtensionRangeOptions { +// optional string label = 20000; +// } +// +// The formatted result is shown below: +// +// extend .google.protobuf./* in-line comment */ExtensionRangeOptions { +// optional string label = 20000; +// } +func (f *formatter) writeInlineComments(comments ast.Comments) { + for i := 0; i < comments.Len(); i++ { + if i > 0 || comments.Index(i).LeadingWhitespace() != "" || f.lastWritten == ';' || f.lastWritten == '}' { + f.Space() + } + text := comments.Index(i).RawText() + if strings.HasPrefix(text, "//") { + text = strings.TrimSpace(strings.TrimPrefix(text, "//")) + text = "/* " + text + " */" + } else { + // no multi-line comments + lines := strings.Split(text, "\n") + for i := range lines { + lines[i] = strings.TrimSpace(lines[i]) + } + text = strings.Join(lines, " ") + } + f.WriteString(text) + } +} + +// writeTrailingEndComments writes the given comments at the end of a line and +// preserves the comment style. This is useful or writing comments attached to +// things like ';' and other tokens that conclude a type definition on a single +// line. +// +// If there is a newline between this trailing comment and the previous node, the +// comments are written immediately underneath the node on a newline. +// +// For example, +// +// enum Type { +// TYPE_UNSPECIFIED = 0; +// } +// // This comment is attached to the '}' +// // So is this one. +func (f *formatter) writeTrailingEndComments(comments ast.Comments) { + for i := 0; i < comments.Len(); i++ { + comment := comments.Index(i) + if i > 0 || comment.LeadingWhitespace() != "" { + f.Space() + } + f.writeComment(comment.RawText()) + } + f.P("") +} + +func (f *formatter) writeComment(comment string) { + if strings.HasPrefix(comment, "/*") && newlineCount(comment) > 0 { + lines := strings.Split(comment, "\n") + // find minimum indent, so we can make all other lines relative to that + minIndent := -1 // sentinel that means unset + // start at 1 because line at index zero starts with "/*", not whitespace + var prefix string + for i := 1; i < len(lines); i++ { + indent, ok := computeIndent(lines[i]) + if ok && (minIndent == -1 || indent < minIndent) { + minIndent = indent + } + if i > 1 && len(prefix) == 0 { + // no shared prefix + continue + } + line := strings.TrimSpace(lines[i]) + if line == "*/" { + continue + } + var linePrefix string + if len(line) > 0 && isCommentPrefix(line[0]) { + linePrefix = line[:1] + } + if i == 1 { + prefix = linePrefix + } else if linePrefix != prefix { + // they do not share prefix + prefix = "" + } + } + if minIndent < 0 { + // This shouldn't be necessary. + // But we do it just in case, to avoid possible panic + minIndent = 0 + } + for i, line := range lines { + trimmedLine := strings.TrimSpace(line) + if trimmedLine == "" || trimmedLine == "*/" || len(prefix) > 0 { + line = trimmedLine + } else { + // we only trim space from the right; for the left, + // we unindent based on indentation found above. + line = unindent(line, minIndent) + line = strings.TrimRightFunc(line, unicode.IsSpace) + } + // If we have a block comment with no prefix, we'll format + // like so: + + /* + This is a multi-line comment example. + It has no comment prefix on each line. + */ + + // But if there IS a prefix, "|" for example, we'll left-align + // the prefix symbol under the asterisk of the comment start + // like this: + + /* + | This comment has a prefix before each line. + | Usually the prefix is asterisk, but it's a + | pipe in this example. + */ + + // Finally, if the comment prefix is an asterisk, we'll left-align + // the comment end so its asterisk also aligns, like so: + + /* + * This comment has a prefix before each line. + * Usually the prefix is asterisk, which is the + * case in this example. + */ + + if i > 0 && line != "*/" { + if len(prefix) == 0 { + line = " " + line + } else { + line = " " + line + } + } + if line == "*/" && prefix == "*" { + // align the comment end with the other asterisks + line = " " + line + } + + if i != len(lines)-1 { + f.P(line) + } else { + // for last line, we don't use P because we don't + // want to print a trailing newline + f.Indent(nil) + f.WriteString(line) + } + } + } else { + f.Indent(nil) + f.WriteString(strings.TrimSpace(comment)) + } +} + +func isCommentPrefix(ch byte) bool { + r := rune(ch) + // A multi-line comment prefix is *usually* an asterisk, like in the following + /* + * Foo + * Bar + * Baz + */ + // But we'll allow other prefixes. But if it's a letter or number, it's not a prefix. + return !unicode.IsLetter(r) && !unicode.IsNumber(r) +} + +func unindent(s string, unindent int) string { + pos := 0 + for i, r := range s { + if pos == unindent { + return s[i:] + } + if pos > unindent { + // removing tab-stop unindented too far, so we + // add back some spaces to compensate + return strings.Repeat(" ", pos-unindent) + s[i:] + } + + switch r { + case ' ': + pos++ + case '\t': + // jump to next tab stop + pos += 8 - (pos % 8) + default: + return s[i:] + } + } + // nothing but whitespace... + return "" +} + +func computeIndent(s string) (int, bool) { + if strings.TrimSpace(s) == "*/" { + return 0, false + } + indent := 0 + for _, r := range s { + switch r { + case ' ': + indent++ + case '\t': + // jump to next tab stop + indent += 8 - (indent % 8) + default: + return indent, true + } + } + // if we get here, line is nothing but whitespace + return 0, false +} + +func (f *formatter) leadingCommentsContainBlankLine(n ast.Node) bool { + info := f.nodeInfo(n) + comments := info.LeadingComments() + for i := 0; i < comments.Len(); i++ { + if newlineCount(comments.Index(i).LeadingWhitespace()) > 1 { + return true + } + } + return newlineCount(info.LeadingWhitespace()) > 1 +} + +func (f *formatter) importHasComment(importNode *ast.ImportNode) bool { + if f.nodeHasComment(importNode) { + return true + } + if importNode == nil { + return false + } + + return f.nodeHasComment(importNode.Keyword) || + f.nodeHasComment(importNode.Name) || + f.nodeHasComment(importNode.Semicolon) || + f.nodeHasComment(importNode.Public) || + f.nodeHasComment(importNode.Weak) +} + +func (f *formatter) nodeHasComment(node ast.Node) bool { + // when node != nil, node's value could be nil, see: https://go.dev/doc/faq#nil_error + if node == nil || reflect.ValueOf(node).IsNil() { + return false + } + + nodeinfo := f.nodeInfo(node) + return nodeinfo.LeadingComments().Len() > 0 || + nodeinfo.TrailingComments().Len() > 0 +} + +func (f *formatter) setTrailingComments(node ast.Node, comments ast.Comments) { + f.overrideTrailingComments[node] = comments +} + +func (f *formatter) nodeInfo(node ast.Node) nodeInfo { + info := f.fileNode.NodeInfo(node) + if trailingComments, ok := f.overrideTrailingComments[node]; ok { + return infoWithTrailingComments{info, trailingComments} + } + return info +} + +type nodeInfo interface { + Start() ast.SourcePos + End() ast.SourcePos + LeadingComments() ast.Comments + TrailingComments() ast.Comments + LeadingWhitespace() string + RawText() string +} + +type infoWithTrailingComments struct { + ast.NodeInfo + trailing ast.Comments +} + +func (n infoWithTrailingComments) TrailingComments() ast.Comments { + return n.trailing +} + +// importSortOrder maps import types to a sort order number, so it can be compared and sorted. +// `import`=3, `import public`=2, `import weak`=1 +func importSortOrder(node *ast.ImportNode) int { + switch { + case node.Public != nil: + return 2 + case node.Weak != nil: + return 1 + default: + return 3 + } +} + +// stringForOptionName returns the string representation of the given option name node. +// This is used for sorting file-level options. +func stringForOptionName(optionNameNode *ast.OptionNameNode) string { + var result string + for j, part := range optionNameNode.Parts { + if j > 0 { + // Add a dot between each of the parts. + result += "." + } + result += stringForFieldReference(part) + } + return result +} + +// stringForFieldReference returns the string representation of the given field reference. +// This is used for sorting file-level options. +func stringForFieldReference(fieldReference *ast.FieldReferenceNode) string { + var result string + if fieldReference.Open != nil { + result += "(" + } + result += string(fieldReference.Name.AsIdentifier()) + if fieldReference.Close != nil { + result += ")" + } + return result +} + +// isOpenBrace returns true if the given node represents one of the +// possible open brace tokens, namely '{', '[', or '<'. +func isOpenBrace(node ast.Node) bool { + if node == nil { + return false + } + runeNode, ok := node.(*ast.RuneNode) + if !ok { + return false + } + return runeNode.Rune == '{' || runeNode.Rune == '[' || runeNode.Rune == '<' +} + +// newlineCount returns the number of newlines in the given value. +// This is useful for determining whether or not we should preserve +// the newline between nodes. +// +// The newlines don't need to be adjacent to each other - all of the +// tokens between them are other whitespace characters, so we can +// safely ignore them. +func newlineCount(value string) int { + return strings.Count(value, "\n") +} + +func messageLiteralOpen(msg *ast.MessageLiteralNode) *ast.RuneNode { + node := msg.Open + if node.Rune == '{' { + return node + } + // If it's not "{" then this message literal used "<" and ">" to enclose it. + // For consistent formatted output, change it to "{". + return ast.NewRuneNode('{', node.Token()) +} + +func messageLiteralClose(msg *ast.MessageLiteralNode) *ast.RuneNode { + node := msg.Close + if node.Rune == '}' { + return node + } + // If it's not "}" then this message literal used "<" and ">" to enclose it. + // For consistent formatted output, change it to "}". + return ast.NewRuneNode('}', node.Token()) +} diff --git a/go/protopace/formatter_test.go b/go/protopace/formatter_test.go new file mode 100644 index 000000000..4ec402c2f --- /dev/null +++ b/go/protopace/formatter_test.go @@ -0,0 +1,26 @@ +package main + +import ( + "os" + "testing" + + s "github.com/Aiven-Open/karapace/go/protopace/schema" + "github.com/stretchr/testify/assert" +) + +func TestFormat(t *testing.T) { + assert := assert.New(t) + + data, _ := os.ReadFile("./fixtures/dependency.proto") + dependencySchema, err := s.FromString("my/awesome/customer/v1/nested_value.proto", string(data), nil) + assert.NoError(err) + assert.NotNil(dependencySchema) + + data, _ = os.ReadFile("./fixtures/test.proto") + testSchema, err := s.FromString("test.proto", string(data), []s.Schema{*dependencySchema}) + assert.NoError(err) + assert.NotNil(testSchema) + + _, err = Format(*testSchema) + assert.NoError(err) +} diff --git a/go/protopace/go.mod b/go/protopace/go.mod new file mode 100644 index 000000000..b24f9651a --- /dev/null +++ b/go/protopace/go.mod @@ -0,0 +1,40 @@ +module github.com/Aiven-Open/karapace/go/protopace + +go 1.22 + +replace github.com/bufbuild/buf => github.com/keejon/buf v0.0.0-20240709110257-34d8b868af21 + +require ( + github.com/bufbuild/buf v1.34.0 + github.com/bufbuild/protocompile v0.14.0 + github.com/gofrs/uuid/v5 v5.2.0 + github.com/stretchr/testify v1.9.0 + go.uber.org/multierr v1.11.0 + go.uber.org/zap v1.27.0 + google.golang.org/protobuf v1.34.2 +) + +require ( + buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240508200655-46a4cf4ba109.2 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/bufbuild/protoplugin v0.0.0-20240323223605-e2735f6c31ee // indirect + github.com/bufbuild/protovalidate-go v0.6.3 // indirect + github.com/bufbuild/protoyaml-go v0.1.9 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/google/cel-go v0.20.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect + go.opentelemetry.io/otel v1.28.0 // indirect + go.opentelemetry.io/otel/trace v1.28.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + golang.org/x/crypto v0.25.0 // indirect + golang.org/x/exp v0.0.0-20240707233637-46b078467d37 // indirect + golang.org/x/mod v0.19.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/text v0.16.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240708141625-4ad9e859172b // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240708141625-4ad9e859172b // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go/protopace/go.sum b/go/protopace/go.sum new file mode 100644 index 000000000..a7959b131 --- /dev/null +++ b/go/protopace/go.sum @@ -0,0 +1,83 @@ +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240508200655-46a4cf4ba109.2 h1:cFrEG/pJch6t62+jqndcPXeTNkYcztS4tBRgNkR+drw= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240508200655-46a4cf4ba109.2/go.mod h1:ylS4c28ACSI59oJrOdW4pHS4n0Hw4TgSPHn8rpHl4Yw= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= +github.com/bufbuild/protocompile v0.14.0 h1:z3DW4IvXE5G/uTOnSQn+qwQQxvhckkTWLS/0No/o7KU= +github.com/bufbuild/protocompile v0.14.0/go.mod h1:N6J1NYzkspJo3ZwyL4Xjvli86XOj1xq4qAasUFxGups= +github.com/bufbuild/protoplugin v0.0.0-20240323223605-e2735f6c31ee h1:E6ET8YUcYJ1lAe6ctR3as7yqzW2BNItDFnaB5zQq/8M= +github.com/bufbuild/protoplugin v0.0.0-20240323223605-e2735f6c31ee/go.mod h1:HjGFxsck9RObrTJp2hXQZfWhPgZqnR6sR1U5fCA/Kus= +github.com/bufbuild/protovalidate-go v0.6.3 h1:wxQyzW035zM16Binbaz/nWAzS12dRIXhZdSUWRY7Fv0= +github.com/bufbuild/protovalidate-go v0.6.3/go.mod h1:J4PtwP9Z2YAGgB0+o+tTWEDtLtXvz/gfhFZD8pbzM/U= +github.com/bufbuild/protoyaml-go v0.1.9 h1:anV5UtF1Mlvkkgp4NWA6U/zOnJFng8Orq4Vf3ZUQHBU= +github.com/bufbuild/protoyaml-go v0.1.9/go.mod h1:KCBItkvZOK/zwGueLdH1Wx1RLyFn5rCH7YjQrdty2Wc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= +github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/gofrs/uuid/v5 v5.2.0 h1:qw1GMx6/y8vhVsx626ImfKMuS5CvJmhIKKtuyvfajMM= +github.com/gofrs/uuid/v5 v5.2.0/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8= +github.com/google/cel-go v0.20.1 h1:nDx9r8S3L4pE61eDdt8igGj8rf5kjYR3ILxWIpWNi84= +github.com/google/cel-go v0.20.1/go.mod h1:kWcIzTsPX0zmQ+H3TirHstLLf9ep5QTsZBN9u4dOYLg= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/keejon/buf v0.0.0-20240709110257-34d8b868af21 h1:9vFdXQGe/i72nN0UAF4gx3TlGuUzHouayOYB1WLW8C0= +github.com/keejon/buf v0.0.0-20240709110257-34d8b868af21/go.mod h1:hgAhSorSpAZGiF0Sqgqc9nhKsrq2bZn8A1mdeSNsVSk= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/exp v0.0.0-20240707233637-46b078467d37 h1:uLDX+AfeFCct3a2C7uIWBKMJIR3CJMhcgfrUAqjRK6w= +golang.org/x/exp v0.0.0-20240707233637-46b078467d37/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= +golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +google.golang.org/genproto/googleapis/api v0.0.0-20240708141625-4ad9e859172b h1:y/kpOWeX2pWERnbsvh/hF+Zmo69wVmjyZhstreXQQeA= +google.golang.org/genproto/googleapis/api v0.0.0-20240708141625-4ad9e859172b/go.mod h1:mw8MG/Qz5wfgYr6VqVCiZcHe/GJEfI+oGGDCohaVgB0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240708141625-4ad9e859172b h1:04+jVzTs2XBnOZcPsLnmrTGqltqJbZQ1Ey26hjYdQQ0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240708141625-4ad9e859172b/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go/protopace/main.go b/go/protopace/main.go new file mode 100644 index 000000000..f239730d2 --- /dev/null +++ b/go/protopace/main.go @@ -0,0 +1,89 @@ +package main + +/* +#include + +struct result { + char* res; + char* err; +}; + +*/ +import "C" +import ( + "fmt" + "unsafe" + + s "github.com/Aiven-Open/karapace/go/protopace/schema" +) + +var all string = "" + +//export SayHello +func SayHello(name *C.char) { + all = all + " " + C.GoString(name) + fmt.Printf("Hello, %s!\n", all) +} + +func result(schema string, err error) *C.struct_result { + res := (*C.struct_result)(C.malloc(C.size_t(unsafe.Sizeof(C.struct_result{})))) + res.res = C.CString(schema) + res.err = nil + if err != nil { + res.err = C.CString(err.Error()) + } + return res +} + +func createSchema(cSchemaName *C.char, cSchema *C.char, cDependencyNames **C.char, cDependencies **C.char, depsLenght C.int) (*s.Schema, error) { + depArray := unsafe.Slice(cDependencies, depsLenght) + depNamesArray := unsafe.Slice(cDependencyNames, depsLenght) + dependencies := []s.Schema{} + for i, dep := range depArray { + dependency, err := s.FromString(C.GoString(depNamesArray[i]), C.GoString(dep), []s.Schema{}) + if err != nil { + return nil, err + } + dependencies = append(dependencies, *dependency) + } + + schema, err := s.FromString(C.GoString(cSchemaName), C.GoString(cSchema), dependencies) + return schema, err +} + +//export FormatSchema +func FormatSchema(cSchemaName *C.char, cSchema *C.char, cDependencyNames **C.char, cDependencies **C.char, depsLenght C.int) *C.struct_result { + schema, err := createSchema(cSchemaName, cSchema, cDependencyNames, cDependencies, depsLenght) + if err != nil { + return result("", err) + } + + res, err := Format(*schema) + if err != nil { + return result("", err) + } + return result(res.Schema, err) +} + +//export CheckCompatibility +func CheckCompatibility( + cSchemaName *C.char, cSchema *C.char, cDependencyNames **C.char, cDependencies **C.char, depsLenght C.int, + cSchemaNamePrev *C.char, cSchemaPrev *C.char, cDependencyNamesPrev **C.char, cDependenciesPrev **C.char, depsLenghtPrev C.int) *C.char { + + schema, err := createSchema(cSchemaName, cSchema, cDependencyNames, cDependencies, depsLenght) + if err != nil { + return C.CString(err.Error()) + } + prevSchema, err := createSchema(cSchemaNamePrev, cSchemaPrev, cDependencyNamesPrev, cDependenciesPrev, depsLenghtPrev) + if err != nil { + return C.CString(err.Error()) + } + + err = Check(*schema, *prevSchema) + if err != nil { + return C.CString(err.Error()) + } + return nil +} + +func main() {} diff --git a/go/protopace/schema/compiler.go b/go/protopace/schema/compiler.go new file mode 100644 index 000000000..5311d022c --- /dev/null +++ b/go/protopace/schema/compiler.go @@ -0,0 +1,10 @@ +package schema + +import ( + "github.com/bufbuild/protocompile" +) + +func NewCompiler(resolver protocompile.Resolver) *protocompile.Compiler { + compiler := protocompile.Compiler{Resolver: resolver, RetainASTs: true} + return &compiler +} diff --git a/go/protopace/schema/google/type/README.md b/go/protopace/schema/google/type/README.md new file mode 100644 index 000000000..adf1563a8 --- /dev/null +++ b/go/protopace/schema/google/type/README.md @@ -0,0 +1,7 @@ +## Google Common Types + +This package contains definitions of common types for Google APIs. +All types defined in this package are suitable for different APIs to +exchange data, and will never break binary compatibility. They should +have design quality comparable to major programming languages like +Java and C#. diff --git a/go/protopace/schema/google/type/calendar_period.proto b/go/protopace/schema/google/type/calendar_period.proto new file mode 100644 index 000000000..25a8f6441 --- /dev/null +++ b/go/protopace/schema/google/type/calendar_period.proto @@ -0,0 +1,56 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/calendarperiod;calendarperiod"; +option java_multiple_files = true; +option java_outer_classname = "CalendarPeriodProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// A `CalendarPeriod` represents the abstract concept of a time period that has +// a canonical start. Grammatically, "the start of the current +// `CalendarPeriod`." All calendar times begin at midnight UTC. +enum CalendarPeriod { + // Undefined period, raises an error. + CALENDAR_PERIOD_UNSPECIFIED = 0; + + // A day. + DAY = 1; + + // A week. Weeks begin on Monday, following + // [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date). + WEEK = 2; + + // A fortnight. The first calendar fortnight of the year begins at the start + // of week 1 according to + // [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date). + FORTNIGHT = 3; + + // A month. + MONTH = 4; + + // A quarter. Quarters start on dates 1-Jan, 1-Apr, 1-Jul, and 1-Oct of each + // year. + QUARTER = 5; + + // A half-year. Half-years start on dates 1-Jan and 1-Jul. + HALF = 6; + + // A year. + YEAR = 7; +} diff --git a/go/protopace/schema/google/type/color.proto b/go/protopace/schema/google/type/color.proto new file mode 100644 index 000000000..3e57c1fb2 --- /dev/null +++ b/go/protopace/schema/google/type/color.proto @@ -0,0 +1,174 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +import "google/protobuf/wrappers.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/color;color"; +option java_multiple_files = true; +option java_outer_classname = "ColorProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a color in the RGBA color space. This representation is designed +// for simplicity of conversion to/from color representations in various +// languages over compactness. For example, the fields of this representation +// can be trivially provided to the constructor of `java.awt.Color` in Java; it +// can also be trivially provided to UIColor's `+colorWithRed:green:blue:alpha` +// method in iOS; and, with just a little work, it can be easily formatted into +// a CSS `rgba()` string in JavaScript. +// +// This reference page doesn't carry information about the absolute color +// space +// that should be used to interpret the RGB value (e.g. sRGB, Adobe RGB, +// DCI-P3, BT.2020, etc.). By default, applications should assume the sRGB color +// space. +// +// When color equality needs to be decided, implementations, unless +// documented otherwise, treat two colors as equal if all their red, +// green, blue, and alpha values each differ by at most 1e-5. +// +// Example (Java): +// +// import com.google.type.Color; +// +// // ... +// public static java.awt.Color fromProto(Color protocolor) { +// float alpha = protocolor.hasAlpha() +// ? protocolor.getAlpha().getValue() +// : 1.0; +// +// return new java.awt.Color( +// protocolor.getRed(), +// protocolor.getGreen(), +// protocolor.getBlue(), +// alpha); +// } +// +// public static Color toProto(java.awt.Color color) { +// float red = (float) color.getRed(); +// float green = (float) color.getGreen(); +// float blue = (float) color.getBlue(); +// float denominator = 255.0; +// Color.Builder resultBuilder = +// Color +// .newBuilder() +// .setRed(red / denominator) +// .setGreen(green / denominator) +// .setBlue(blue / denominator); +// int alpha = color.getAlpha(); +// if (alpha != 255) { +// result.setAlpha( +// FloatValue +// .newBuilder() +// .setValue(((float) alpha) / denominator) +// .build()); +// } +// return resultBuilder.build(); +// } +// // ... +// +// Example (iOS / Obj-C): +// +// // ... +// static UIColor* fromProto(Color* protocolor) { +// float red = [protocolor red]; +// float green = [protocolor green]; +// float blue = [protocolor blue]; +// FloatValue* alpha_wrapper = [protocolor alpha]; +// float alpha = 1.0; +// if (alpha_wrapper != nil) { +// alpha = [alpha_wrapper value]; +// } +// return [UIColor colorWithRed:red green:green blue:blue alpha:alpha]; +// } +// +// static Color* toProto(UIColor* color) { +// CGFloat red, green, blue, alpha; +// if (![color getRed:&red green:&green blue:&blue alpha:&alpha]) { +// return nil; +// } +// Color* result = [[Color alloc] init]; +// [result setRed:red]; +// [result setGreen:green]; +// [result setBlue:blue]; +// if (alpha <= 0.9999) { +// [result setAlpha:floatWrapperWithValue(alpha)]; +// } +// [result autorelease]; +// return result; +// } +// // ... +// +// Example (JavaScript): +// +// // ... +// +// var protoToCssColor = function(rgb_color) { +// var redFrac = rgb_color.red || 0.0; +// var greenFrac = rgb_color.green || 0.0; +// var blueFrac = rgb_color.blue || 0.0; +// var red = Math.floor(redFrac * 255); +// var green = Math.floor(greenFrac * 255); +// var blue = Math.floor(blueFrac * 255); +// +// if (!('alpha' in rgb_color)) { +// return rgbToCssColor(red, green, blue); +// } +// +// var alphaFrac = rgb_color.alpha.value || 0.0; +// var rgbParams = [red, green, blue].join(','); +// return ['rgba(', rgbParams, ',', alphaFrac, ')'].join(''); +// }; +// +// var rgbToCssColor = function(red, green, blue) { +// var rgbNumber = new Number((red << 16) | (green << 8) | blue); +// var hexString = rgbNumber.toString(16); +// var missingZeros = 6 - hexString.length; +// var resultBuilder = ['#']; +// for (var i = 0; i < missingZeros; i++) { +// resultBuilder.push('0'); +// } +// resultBuilder.push(hexString); +// return resultBuilder.join(''); +// }; +// +// // ... +message Color { + // The amount of red in the color as a value in the interval [0, 1]. + float red = 1; + + // The amount of green in the color as a value in the interval [0, 1]. + float green = 2; + + // The amount of blue in the color as a value in the interval [0, 1]. + float blue = 3; + + // The fraction of this color that should be applied to the pixel. That is, + // the final pixel color is defined by the equation: + // + // `pixel color = alpha * (this color) + (1.0 - alpha) * (background color)` + // + // This means that a value of 1.0 corresponds to a solid color, whereas + // a value of 0.0 corresponds to a completely transparent color. This + // uses a wrapper message rather than a simple float scalar so that it is + // possible to distinguish between a default value and the value being unset. + // If omitted, this color object is rendered as a solid color + // (as if the alpha value had been explicitly given a value of 1.0). + google.protobuf.FloatValue alpha = 4; +} diff --git a/go/protopace/schema/google/type/date.proto b/go/protopace/schema/google/type/date.proto new file mode 100644 index 000000000..6370cd869 --- /dev/null +++ b/go/protopace/schema/google/type/date.proto @@ -0,0 +1,52 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/date;date"; +option java_multiple_files = true; +option java_outer_classname = "DateProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a whole or partial calendar date, such as a birthday. The time of +// day and time zone are either specified elsewhere or are insignificant. The +// date is relative to the Gregorian Calendar. This can represent one of the +// following: +// +// * A full date, with non-zero year, month, and day values +// * A month and day value, with a zero year, such as an anniversary +// * A year on its own, with zero month and day values +// * A year and month value, with a zero day, such as a credit card expiration +// date +// +// Related types are [google.type.TimeOfDay][google.type.TimeOfDay] and +// `google.protobuf.Timestamp`. +message Date { + // Year of the date. Must be from 1 to 9999, or 0 to specify a date without + // a year. + int32 year = 1; + + // Month of a year. Must be from 1 to 12, or 0 to specify a year without a + // month and day. + int32 month = 2; + + // Day of a month. Must be from 1 to 31 and valid for the year and month, or 0 + // to specify a year by itself or a year and month where the day isn't + // significant. + int32 day = 3; +} diff --git a/go/protopace/schema/google/type/datetime.proto b/go/protopace/schema/google/type/datetime.proto new file mode 100644 index 000000000..a363a41ef --- /dev/null +++ b/go/protopace/schema/google/type/datetime.proto @@ -0,0 +1,104 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +import "google/protobuf/duration.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/datetime;datetime"; +option java_multiple_files = true; +option java_outer_classname = "DateTimeProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents civil time (or occasionally physical time). +// +// This type can represent a civil time in one of a few possible ways: +// +// * When utc_offset is set and time_zone is unset: a civil time on a calendar +// day with a particular offset from UTC. +// * When time_zone is set and utc_offset is unset: a civil time on a calendar +// day in a particular time zone. +// * When neither time_zone nor utc_offset is set: a civil time on a calendar +// day in local time. +// +// The date is relative to the Proleptic Gregorian Calendar. +// +// If year is 0, the DateTime is considered not to have a specific year. month +// and day must have valid, non-zero values. +// +// This type may also be used to represent a physical time if all the date and +// time fields are set and either case of the `time_offset` oneof is set. +// Consider using `Timestamp` message for physical time instead. If your use +// case also would like to store the user's timezone, that can be done in +// another field. +// +// This type is more flexible than some applications may want. Make sure to +// document and validate your application's limitations. +message DateTime { + // Optional. Year of date. Must be from 1 to 9999, or 0 if specifying a + // datetime without a year. + int32 year = 1; + + // Required. Month of year. Must be from 1 to 12. + int32 month = 2; + + // Required. Day of month. Must be from 1 to 31 and valid for the year and + // month. + int32 day = 3; + + // Required. Hours of day in 24 hour format. Should be from 0 to 23. An API + // may choose to allow the value "24:00:00" for scenarios like business + // closing time. + int32 hours = 4; + + // Required. Minutes of hour of day. Must be from 0 to 59. + int32 minutes = 5; + + // Required. Seconds of minutes of the time. Must normally be from 0 to 59. An + // API may allow the value 60 if it allows leap-seconds. + int32 seconds = 6; + + // Required. Fractions of seconds in nanoseconds. Must be from 0 to + // 999,999,999. + int32 nanos = 7; + + // Optional. Specifies either the UTC offset or the time zone of the DateTime. + // Choose carefully between them, considering that time zone data may change + // in the future (for example, a country modifies their DST start/end dates, + // and future DateTimes in the affected range had already been stored). + // If omitted, the DateTime is considered to be in local time. + oneof time_offset { + // UTC offset. Must be whole seconds, between -18 hours and +18 hours. + // For example, a UTC offset of -4:00 would be represented as + // { seconds: -14400 }. + google.protobuf.Duration utc_offset = 8; + + // Time zone. + TimeZone time_zone = 9; + } +} + +// Represents a time zone from the +// [IANA Time Zone Database](https://www.iana.org/time-zones). +message TimeZone { + // IANA Time Zone Database time zone, e.g. "America/New_York". + string id = 1; + + // Optional. IANA Time Zone Database version number, e.g. "2019a". + string version = 2; +} diff --git a/go/protopace/schema/google/type/dayofweek.proto b/go/protopace/schema/google/type/dayofweek.proto new file mode 100644 index 000000000..e16c19469 --- /dev/null +++ b/go/protopace/schema/google/type/dayofweek.proto @@ -0,0 +1,50 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/dayofweek;dayofweek"; +option java_multiple_files = true; +option java_outer_classname = "DayOfWeekProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a day of the week. +enum DayOfWeek { + // The day of the week is unspecified. + DAY_OF_WEEK_UNSPECIFIED = 0; + + // Monday + MONDAY = 1; + + // Tuesday + TUESDAY = 2; + + // Wednesday + WEDNESDAY = 3; + + // Thursday + THURSDAY = 4; + + // Friday + FRIDAY = 5; + + // Saturday + SATURDAY = 6; + + // Sunday + SUNDAY = 7; +} diff --git a/go/protopace/schema/google/type/decimal.proto b/go/protopace/schema/google/type/decimal.proto new file mode 100644 index 000000000..293d08273 --- /dev/null +++ b/go/protopace/schema/google/type/decimal.proto @@ -0,0 +1,95 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/decimal;decimal"; +option java_multiple_files = true; +option java_outer_classname = "DecimalProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// A representation of a decimal value, such as 2.5. Clients may convert values +// into language-native decimal formats, such as Java's [BigDecimal][] or +// Python's [decimal.Decimal][]. +// +// [BigDecimal]: +// https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/math/BigDecimal.html +// [decimal.Decimal]: https://docs.python.org/3/library/decimal.html +message Decimal { + // The decimal value, as a string. + // + // The string representation consists of an optional sign, `+` (`U+002B`) + // or `-` (`U+002D`), followed by a sequence of zero or more decimal digits + // ("the integer"), optionally followed by a fraction, optionally followed + // by an exponent. + // + // The fraction consists of a decimal point followed by zero or more decimal + // digits. The string must contain at least one digit in either the integer + // or the fraction. The number formed by the sign, the integer and the + // fraction is referred to as the significand. + // + // The exponent consists of the character `e` (`U+0065`) or `E` (`U+0045`) + // followed by one or more decimal digits. + // + // Services **should** normalize decimal values before storing them by: + // + // - Removing an explicitly-provided `+` sign (`+2.5` -> `2.5`). + // - Replacing a zero-length integer value with `0` (`.5` -> `0.5`). + // - Coercing the exponent character to lower-case (`2.5E8` -> `2.5e8`). + // - Removing an explicitly-provided zero exponent (`2.5e0` -> `2.5`). + // + // Services **may** perform additional normalization based on its own needs + // and the internal decimal implementation selected, such as shifting the + // decimal point and exponent value together (example: `2.5e-1` <-> `0.25`). + // Additionally, services **may** preserve trailing zeroes in the fraction + // to indicate increased precision, but are not required to do so. + // + // Note that only the `.` character is supported to divide the integer + // and the fraction; `,` **should not** be supported regardless of locale. + // Additionally, thousand separators **should not** be supported. If a + // service does support them, values **must** be normalized. + // + // The ENBF grammar is: + // + // DecimalString = + // [Sign] Significand [Exponent]; + // + // Sign = '+' | '-'; + // + // Significand = + // Digits ['.'] [Digits] | [Digits] '.' Digits; + // + // Exponent = ('e' | 'E') [Sign] Digits; + // + // Digits = { '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' }; + // + // Services **should** clearly document the range of supported values, the + // maximum supported precision (total number of digits), and, if applicable, + // the scale (number of digits after the decimal point), as well as how it + // behaves when receiving out-of-bounds values. + // + // Services **may** choose to accept values passed as input even when the + // value has a higher precision or scale than the service supports, and + // **should** round the value to fit the supported scale. Alternatively, the + // service **may** error with `400 Bad Request` (`INVALID_ARGUMENT` in gRPC) + // if precision would be lost. + // + // Services **should** error with `400 Bad Request` (`INVALID_ARGUMENT` in + // gRPC) if the service receives a value outside of the supported range. + string value = 1; +} diff --git a/go/protopace/schema/google/type/expr.proto b/go/protopace/schema/google/type/expr.proto new file mode 100644 index 000000000..544e66874 --- /dev/null +++ b/go/protopace/schema/google/type/expr.proto @@ -0,0 +1,73 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/expr;expr"; +option java_multiple_files = true; +option java_outer_classname = "ExprProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a textual expression in the Common Expression Language (CEL) +// syntax. CEL is a C-like expression language. The syntax and semantics of CEL +// are documented at https://github.com/google/cel-spec. +// +// Example (Comparison): +// +// title: "Summary size limit" +// description: "Determines if a summary is less than 100 chars" +// expression: "document.summary.size() < 100" +// +// Example (Equality): +// +// title: "Requestor is owner" +// description: "Determines if requestor is the document owner" +// expression: "document.owner == request.auth.claims.email" +// +// Example (Logic): +// +// title: "Public documents" +// description: "Determine whether the document should be publicly visible" +// expression: "document.type != 'private' && document.type != 'internal'" +// +// Example (Data Manipulation): +// +// title: "Notification string" +// description: "Create a notification string with a timestamp." +// expression: "'New message received at ' + string(document.create_time)" +// +// The exact variables and functions that may be referenced within an expression +// are determined by the service that evaluates it. See the service +// documentation for additional information. +message Expr { + // Textual representation of an expression in Common Expression Language + // syntax. + string expression = 1; + + // Optional. Title for the expression, i.e. a short string describing + // its purpose. This can be used e.g. in UIs which allow to enter the + // expression. + string title = 2; + + // Optional. Description of the expression. This is a longer text which + // describes the expression, e.g. when hovered over it in a UI. + string description = 3; + + // Optional. String indicating the location of the expression for error + // reporting, e.g. a file name and a position in the file. + string location = 4; +} diff --git a/go/protopace/schema/google/type/fraction.proto b/go/protopace/schema/google/type/fraction.proto new file mode 100644 index 000000000..06f072322 --- /dev/null +++ b/go/protopace/schema/google/type/fraction.proto @@ -0,0 +1,33 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/fraction;fraction"; +option java_multiple_files = true; +option java_outer_classname = "FractionProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a fraction in terms of a numerator divided by a denominator. +message Fraction { + // The numerator in the fraction, e.g. 2 in 2/3. + int64 numerator = 1; + + // The value by which the numerator is divided, e.g. 3 in 2/3. Must be + // positive. + int64 denominator = 2; +} diff --git a/go/protopace/schema/google/type/interval.proto b/go/protopace/schema/google/type/interval.proto new file mode 100644 index 000000000..fcf94c866 --- /dev/null +++ b/go/protopace/schema/google/type/interval.proto @@ -0,0 +1,46 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +import "google/protobuf/timestamp.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/interval;interval"; +option java_multiple_files = true; +option java_outer_classname = "IntervalProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a time interval, encoded as a Timestamp start (inclusive) and a +// Timestamp end (exclusive). +// +// The start must be less than or equal to the end. +// When the start equals the end, the interval is empty (matches no time). +// When both start and end are unspecified, the interval matches any time. +message Interval { + // Optional. Inclusive start of the interval. + // + // If specified, a Timestamp matching this interval will have to be the same + // or after the start. + google.protobuf.Timestamp start_time = 1; + + // Optional. Exclusive end of the interval. + // + // If specified, a Timestamp matching this interval will have to be before the + // end. + google.protobuf.Timestamp end_time = 2; +} diff --git a/go/protopace/schema/google/type/latlng.proto b/go/protopace/schema/google/type/latlng.proto new file mode 100644 index 000000000..daeba48b4 --- /dev/null +++ b/go/protopace/schema/google/type/latlng.proto @@ -0,0 +1,37 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/latlng;latlng"; +option java_multiple_files = true; +option java_outer_classname = "LatLngProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// An object that represents a latitude/longitude pair. This is expressed as a +// pair of doubles to represent degrees latitude and degrees longitude. Unless +// specified otherwise, this must conform to the +// WGS84 +// standard. Values must be within normalized ranges. +message LatLng { + // The latitude in degrees. It must be in the range [-90.0, +90.0]. + double latitude = 1; + + // The longitude in degrees. It must be in the range [-180.0, +180.0]. + double longitude = 2; +} diff --git a/go/protopace/schema/google/type/localized_text.proto b/go/protopace/schema/google/type/localized_text.proto new file mode 100644 index 000000000..82d083c43 --- /dev/null +++ b/go/protopace/schema/google/type/localized_text.proto @@ -0,0 +1,36 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/localized_text;localized_text"; +option java_multiple_files = true; +option java_outer_classname = "LocalizedTextProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Localized variant of a text in a particular language. +message LocalizedText { + // Localized string in the language corresponding to `language_code' below. + string text = 1; + + // The text's BCP-47 language code, such as "en-US" or "sr-Latn". + // + // For more information, see + // http://www.unicode.org/reports/tr35/#Unicode_locale_identifier. + string language_code = 2; +} diff --git a/go/protopace/schema/google/type/money.proto b/go/protopace/schema/google/type/money.proto new file mode 100644 index 000000000..c61094336 --- /dev/null +++ b/go/protopace/schema/google/type/money.proto @@ -0,0 +1,42 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/money;money"; +option java_multiple_files = true; +option java_outer_classname = "MoneyProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents an amount of money with its currency type. +message Money { + // The three-letter currency code defined in ISO 4217. + string currency_code = 1; + + // The whole units of the amount. + // For example if `currencyCode` is `"USD"`, then 1 unit is one US dollar. + int64 units = 2; + + // Number of nano (10^-9) units of the amount. + // The value must be between -999,999,999 and +999,999,999 inclusive. + // If `units` is positive, `nanos` must be positive or zero. + // If `units` is zero, `nanos` can be positive, zero, or negative. + // If `units` is negative, `nanos` must be negative or zero. + // For example $-1.75 is represented as `units`=-1 and `nanos`=-750,000,000. + int32 nanos = 3; +} diff --git a/go/protopace/schema/google/type/month.proto b/go/protopace/schema/google/type/month.proto new file mode 100644 index 000000000..19982cb51 --- /dev/null +++ b/go/protopace/schema/google/type/month.proto @@ -0,0 +1,65 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/month;month"; +option java_multiple_files = true; +option java_outer_classname = "MonthProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a month in the Gregorian calendar. +enum Month { + // The unspecified month. + MONTH_UNSPECIFIED = 0; + + // The month of January. + JANUARY = 1; + + // The month of February. + FEBRUARY = 2; + + // The month of March. + MARCH = 3; + + // The month of April. + APRIL = 4; + + // The month of May. + MAY = 5; + + // The month of June. + JUNE = 6; + + // The month of July. + JULY = 7; + + // The month of August. + AUGUST = 8; + + // The month of September. + SEPTEMBER = 9; + + // The month of October. + OCTOBER = 10; + + // The month of November. + NOVEMBER = 11; + + // The month of December. + DECEMBER = 12; +} diff --git a/go/protopace/schema/google/type/phone_number.proto b/go/protopace/schema/google/type/phone_number.proto new file mode 100644 index 000000000..370d1623d --- /dev/null +++ b/go/protopace/schema/google/type/phone_number.proto @@ -0,0 +1,113 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/phone_number;phone_number"; +option java_multiple_files = true; +option java_outer_classname = "PhoneNumberProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// An object representing a phone number, suitable as an API wire format. +// +// This representation: +// +// - should not be used for locale-specific formatting of a phone number, such +// as "+1 (650) 253-0000 ext. 123" +// +// - is not designed for efficient storage +// - may not be suitable for dialing - specialized libraries (see references) +// should be used to parse the number for that purpose +// +// To do something meaningful with this number, such as format it for various +// use-cases, convert it to an `i18n.phonenumbers.PhoneNumber` object first. +// +// For instance, in Java this would be: +// +// com.google.type.PhoneNumber wireProto = +// com.google.type.PhoneNumber.newBuilder().build(); +// com.google.i18n.phonenumbers.Phonenumber.PhoneNumber phoneNumber = +// PhoneNumberUtil.getInstance().parse(wireProto.getE164Number(), "ZZ"); +// if (!wireProto.getExtension().isEmpty()) { +// phoneNumber.setExtension(wireProto.getExtension()); +// } +// +// Reference(s): +// - https://github.com/google/libphonenumber +message PhoneNumber { + // An object representing a short code, which is a phone number that is + // typically much shorter than regular phone numbers and can be used to + // address messages in MMS and SMS systems, as well as for abbreviated dialing + // (e.g. "Text 611 to see how many minutes you have remaining on your plan."). + // + // Short codes are restricted to a region and are not internationally + // dialable, which means the same short code can exist in different regions, + // with different usage and pricing, even if those regions share the same + // country calling code (e.g. US and CA). + message ShortCode { + // Required. The BCP-47 region code of the location where calls to this + // short code can be made, such as "US" and "BB". + // + // Reference(s): + // - http://www.unicode.org/reports/tr35/#unicode_region_subtag + string region_code = 1; + + // Required. The short code digits, without a leading plus ('+') or country + // calling code, e.g. "611". + string number = 2; + } + + // Required. Either a regular number, or a short code. New fields may be + // added to the oneof below in the future, so clients should ignore phone + // numbers for which none of the fields they coded against are set. + oneof kind { + // The phone number, represented as a leading plus sign ('+'), followed by a + // phone number that uses a relaxed ITU E.164 format consisting of the + // country calling code (1 to 3 digits) and the subscriber number, with no + // additional spaces or formatting, e.g.: + // - correct: "+15552220123" + // - incorrect: "+1 (555) 222-01234 x123". + // + // The ITU E.164 format limits the latter to 12 digits, but in practice not + // all countries respect that, so we relax that restriction here. + // National-only numbers are not allowed. + // + // References: + // - https://www.itu.int/rec/T-REC-E.164-201011-I + // - https://en.wikipedia.org/wiki/E.164. + // - https://en.wikipedia.org/wiki/List_of_country_calling_codes + string e164_number = 1; + + // A short code. + // + // Reference(s): + // - https://en.wikipedia.org/wiki/Short_code + ShortCode short_code = 2; + } + + // The phone number's extension. The extension is not standardized in ITU + // recommendations, except for being defined as a series of numbers with a + // maximum length of 40 digits. Other than digits, some other dialing + // characters such as ',' (indicating a wait) or '#' may be stored here. + // + // Note that no regions currently use extensions with short codes, so this + // field is normally only set in conjunction with an E.164 number. It is held + // separately from the E.164 number to allow for short code extensions in the + // future. + string extension = 3; +} diff --git a/go/protopace/schema/google/type/postal_address.proto b/go/protopace/schema/google/type/postal_address.proto new file mode 100644 index 000000000..7023a9b3e --- /dev/null +++ b/go/protopace/schema/google/type/postal_address.proto @@ -0,0 +1,134 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/postaladdress;postaladdress"; +option java_multiple_files = true; +option java_outer_classname = "PostalAddressProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a postal address, e.g. for postal delivery or payments addresses. +// Given a postal address, a postal service can deliver items to a premise, P.O. +// Box or similar. +// It is not intended to model geographical locations (roads, towns, +// mountains). +// +// In typical usage an address would be created via user input or from importing +// existing data, depending on the type of process. +// +// Advice on address input / editing: +// - Use an i18n-ready address widget such as +// https://github.com/google/libaddressinput) +// - Users should not be presented with UI elements for input or editing of +// fields outside countries where that field is used. +// +// For more guidance on how to use this schema, please see: +// https://support.google.com/business/answer/6397478 +message PostalAddress { + // The schema revision of the `PostalAddress`. This must be set to 0, which is + // the latest revision. + // + // All new revisions **must** be backward compatible with old revisions. + int32 revision = 1; + + // Required. CLDR region code of the country/region of the address. This + // is never inferred and it is up to the user to ensure the value is + // correct. See http://cldr.unicode.org/ and + // http://www.unicode.org/cldr/charts/30/supplemental/territory_information.html + // for details. Example: "CH" for Switzerland. + string region_code = 2; + + // Optional. BCP-47 language code of the contents of this address (if + // known). This is often the UI language of the input form or is expected + // to match one of the languages used in the address' country/region, or their + // transliterated equivalents. + // This can affect formatting in certain countries, but is not critical + // to the correctness of the data and will never affect any validation or + // other non-formatting related operations. + // + // If this value is not known, it should be omitted (rather than specifying a + // possibly incorrect default). + // + // Examples: "zh-Hant", "ja", "ja-Latn", "en". + string language_code = 3; + + // Optional. Postal code of the address. Not all countries use or require + // postal codes to be present, but where they are used, they may trigger + // additional validation with other parts of the address (e.g. state/zip + // validation in the U.S.A.). + string postal_code = 4; + + // Optional. Additional, country-specific, sorting code. This is not used + // in most regions. Where it is used, the value is either a string like + // "CEDEX", optionally followed by a number (e.g. "CEDEX 7"), or just a number + // alone, representing the "sector code" (Jamaica), "delivery area indicator" + // (Malawi) or "post office indicator" (e.g. Côte d'Ivoire). + string sorting_code = 5; + + // Optional. Highest administrative subdivision which is used for postal + // addresses of a country or region. + // For example, this can be a state, a province, an oblast, or a prefecture. + // Specifically, for Spain this is the province and not the autonomous + // community (e.g. "Barcelona" and not "Catalonia"). + // Many countries don't use an administrative area in postal addresses. E.g. + // in Switzerland this should be left unpopulated. + string administrative_area = 6; + + // Optional. Generally refers to the city/town portion of the address. + // Examples: US city, IT comune, UK post town. + // In regions of the world where localities are not well defined or do not fit + // into this structure well, leave locality empty and use address_lines. + string locality = 7; + + // Optional. Sublocality of the address. + // For example, this can be neighborhoods, boroughs, districts. + string sublocality = 8; + + // Unstructured address lines describing the lower levels of an address. + // + // Because values in address_lines do not have type information and may + // sometimes contain multiple values in a single field (e.g. + // "Austin, TX"), it is important that the line order is clear. The order of + // address lines should be "envelope order" for the country/region of the + // address. In places where this can vary (e.g. Japan), address_language is + // used to make it explicit (e.g. "ja" for large-to-small ordering and + // "ja-Latn" or "en" for small-to-large). This way, the most specific line of + // an address can be selected based on the language. + // + // The minimum permitted structural representation of an address consists + // of a region_code with all remaining information placed in the + // address_lines. It would be possible to format such an address very + // approximately without geocoding, but no semantic reasoning could be + // made about any of the address components until it was at least + // partially resolved. + // + // Creating an address only containing a region_code and address_lines, and + // then geocoding is the recommended way to handle completely unstructured + // addresses (as opposed to guessing which parts of the address should be + // localities or administrative areas). + repeated string address_lines = 9; + + // Optional. The recipient at the address. + // This field may, under certain circumstances, contain multiline information. + // For example, it might contain "care of" information. + repeated string recipients = 10; + + // Optional. The name of the organization at the address. + string organization = 11; +} diff --git a/go/protopace/schema/google/type/quaternion.proto b/go/protopace/schema/google/type/quaternion.proto new file mode 100644 index 000000000..416de30cf --- /dev/null +++ b/go/protopace/schema/google/type/quaternion.proto @@ -0,0 +1,94 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/quaternion;quaternion"; +option java_multiple_files = true; +option java_outer_classname = "QuaternionProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// A quaternion is defined as the quotient of two directed lines in a +// three-dimensional space or equivalently as the quotient of two Euclidean +// vectors (https://en.wikipedia.org/wiki/Quaternion). +// +// Quaternions are often used in calculations involving three-dimensional +// rotations (https://en.wikipedia.org/wiki/Quaternions_and_spatial_rotation), +// as they provide greater mathematical robustness by avoiding the gimbal lock +// problems that can be encountered when using Euler angles +// (https://en.wikipedia.org/wiki/Gimbal_lock). +// +// Quaternions are generally represented in this form: +// +// w + xi + yj + zk +// +// where x, y, z, and w are real numbers, and i, j, and k are three imaginary +// numbers. +// +// Our naming choice `(x, y, z, w)` comes from the desire to avoid confusion for +// those interested in the geometric properties of the quaternion in the 3D +// Cartesian space. Other texts often use alternative names or subscripts, such +// as `(a, b, c, d)`, `(1, i, j, k)`, or `(0, 1, 2, 3)`, which are perhaps +// better suited for mathematical interpretations. +// +// To avoid any confusion, as well as to maintain compatibility with a large +// number of software libraries, the quaternions represented using the protocol +// buffer below *must* follow the Hamilton convention, which defines `ij = k` +// (i.e. a right-handed algebra), and therefore: +// +// i^2 = j^2 = k^2 = ijk = −1 +// ij = −ji = k +// jk = −kj = i +// ki = −ik = j +// +// Please DO NOT use this to represent quaternions that follow the JPL +// convention, or any of the other quaternion flavors out there. +// +// Definitions: +// +// - Quaternion norm (or magnitude): `sqrt(x^2 + y^2 + z^2 + w^2)`. +// - Unit (or normalized) quaternion: a quaternion whose norm is 1. +// - Pure quaternion: a quaternion whose scalar component (`w`) is 0. +// - Rotation quaternion: a unit quaternion used to represent rotation. +// - Orientation quaternion: a unit quaternion used to represent orientation. +// +// A quaternion can be normalized by dividing it by its norm. The resulting +// quaternion maintains the same direction, but has a norm of 1, i.e. it moves +// on the unit sphere. This is generally necessary for rotation and orientation +// quaternions, to avoid rounding errors: +// https://en.wikipedia.org/wiki/Rotation_formalisms_in_three_dimensions +// +// Note that `(x, y, z, w)` and `(-x, -y, -z, -w)` represent the same rotation, +// but normalization would be even more useful, e.g. for comparison purposes, if +// it would produce a unique representation. It is thus recommended that `w` be +// kept positive, which can be achieved by changing all the signs when `w` is +// negative. +// +message Quaternion { + // The x component. + double x = 1; + + // The y component. + double y = 2; + + // The z component. + double z = 3; + + // The scalar component. + double w = 4; +} diff --git a/go/protopace/schema/google/type/timeofday.proto b/go/protopace/schema/google/type/timeofday.proto new file mode 100644 index 000000000..3735745a4 --- /dev/null +++ b/go/protopace/schema/google/type/timeofday.proto @@ -0,0 +1,44 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/timeofday;timeofday"; +option java_multiple_files = true; +option java_outer_classname = "TimeOfDayProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a time of day. The date and time zone are either not significant +// or are specified elsewhere. An API may choose to allow leap seconds. Related +// types are [google.type.Date][google.type.Date] and +// `google.protobuf.Timestamp`. +message TimeOfDay { + // Hours of day in 24 hour format. Should be from 0 to 23. An API may choose + // to allow the value "24:00:00" for scenarios like business closing time. + int32 hours = 1; + + // Minutes of hour of day. Must be from 0 to 59. + int32 minutes = 2; + + // Seconds of minutes of the time. Must normally be from 0 to 59. An API may + // allow the value 60 if it allows leap-seconds. + int32 seconds = 3; + + // Fractions of seconds in nanoseconds. Must be from 0 to 999,999,999. + int32 nanos = 4; +} diff --git a/go/protopace/schema/google/type/type.yaml b/go/protopace/schema/google/type/type.yaml new file mode 100644 index 000000000..d5c71364d --- /dev/null +++ b/go/protopace/schema/google/type/type.yaml @@ -0,0 +1,40 @@ +type: google.api.Service +config_version: 3 +name: type.googleapis.com +title: Common Types + +types: +- name: google.type.Color +- name: google.type.Date +- name: google.type.DateTime +- name: google.type.Decimal +- name: google.type.Expr +- name: google.type.Fraction +- name: google.type.Interval +- name: google.type.LatLng +- name: google.type.LocalizedText +- name: google.type.Money +- name: google.type.PhoneNumber +- name: google.type.PostalAddress +- name: google.type.Quaternion +- name: google.type.TimeOfDay + +enums: +- name: google.type.CalendarPeriod +- name: google.type.DayOfWeek +- name: google.type.Month + +documentation: + summary: Defines common types for Google APIs. + overview: |- + # Google Common Types + + This package contains definitions of common types for Google APIs. + All types defined in this package are suitable for different APIs to + exchange data, and will never break binary compatibility. They should + have design quality comparable to major programming languages like + Java and C#. + + NOTE: Some common types are defined in the package `google.protobuf` + as they are directly supported by Protocol Buffers compiler and + runtime. Those types are called Well-Known Types. diff --git a/go/protopace/schema/google_types.go b/go/protopace/schema/google_types.go new file mode 100644 index 000000000..e81f5a533 --- /dev/null +++ b/go/protopace/schema/google_types.go @@ -0,0 +1,23 @@ +package schema + +import ( + "embed" + "io" + + "github.com/bufbuild/protocompile" +) + +//go:embed google/type/*.proto +var files embed.FS + +// WithGoogleTypeImports returns a new resolver that can provide the source code for google/types protos +func WithGoogleTypeImports(resolver protocompile.Resolver) protocompile.Resolver { + return protocompile.CompositeResolver{ + resolver, + &protocompile.SourceResolver{ + Accessor: func(path string) (io.ReadCloser, error) { + return files.Open(path) + }, + }, + } +} diff --git a/go/protopace/schema/resolver.go b/go/protopace/schema/resolver.go new file mode 100644 index 000000000..9ace3e55d --- /dev/null +++ b/go/protopace/schema/resolver.go @@ -0,0 +1,39 @@ +package schema + +import ( + "fmt" + "strings" + + "github.com/bufbuild/protocompile" +) + +type SchemaResolver struct { + schemas map[string]Schema +} + +func NewSchemaResolver(schemas []Schema) protocompile.Resolver { + schemasIndex := map[string]Schema{} + for _, schema := range schemas { + schemasIndex[schema.Name] = schema + } + resolver := SchemaResolver{schemas: schemasIndex} + return WithGoogleTypeImports(protocompile.WithStandardImports(&resolver)) +} + +func (s *SchemaResolver) AddSchema(schema Schema) { + s.schemas[schema.Name] = schema +} + +// FindFileByPath implements protocompile.Resolver. +func (s *SchemaResolver) FindFileByPath(path string) (protocompile.SearchResult, error) { + searchResult := protocompile.SearchResult{} + schema, ok := s.schemas[path] + if !ok { + return searchResult, fmt.Errorf("schema not found: %s", path) + } + searchResult.Source = strings.NewReader(schema.Schema) + searchResult.ParseResult = schema.ParserResult + return searchResult, nil +} + +var _ protocompile.Resolver = (*SchemaResolver)(nil) diff --git a/go/protopace/schema/schema.go b/go/protopace/schema/schema.go new file mode 100644 index 000000000..1a0bb822e --- /dev/null +++ b/go/protopace/schema/schema.go @@ -0,0 +1,81 @@ +package schema + +import ( + "context" + "strings" + + "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/protocompile/linker" + "github.com/bufbuild/protocompile/parser" + "github.com/bufbuild/protocompile/reporter" + "github.com/gofrs/uuid/v5" +) + +var ( + handler = reporter.NewHandler(nil) +) + +type Schema struct { + Schema string + Name string + ParserResult parser.Result + Dependencies []Schema +} + +func FromString(name string, proto string, dependencies []Schema) (*Schema, error) { + fileNode, err := parser.Parse(name, strings.NewReader(proto), handler) + if err != nil { + return nil, err + } + result, err := parser.ResultFromAST(fileNode, true, handler) + if err != nil { + return nil, err + } + return &Schema{Schema: proto, Name: name, ParserResult: result, Dependencies: dependencies}, nil +} + +func (s Schema) Compile() ([]linker.Result, error) { + resolver := NewSchemaResolver(append(s.Dependencies, s)) + compiler := NewCompiler(resolver) + ctx := context.Background() + schemas := []string{s.Name} + for _, dep := range s.Dependencies { + schemas = append(schemas, dep.Name) + } + files, err := compiler.Compile(ctx, schemas...) + if err != nil { + return nil, err + } + res := make([]linker.Result, len(files)) + for i, f := range files { + res[i] = f.(linker.Result) + } + return res, nil +} + +func (s Schema) CompileBufImage() (bufimage.Image, error) { + res, err := s.Compile() + if err != nil { + return nil, err + } + files := make([]bufimage.ImageFile, len(res)) + for i, r := range res { + file, err := bufimage.NewImageFile( + r.FileDescriptorProto(), + nil, + uuid.Nil, + "", + "", + false, + false, + nil, + ) + if err != nil { + return nil, err + } + files[i] = file + } + + image, err := bufimage.NewImage(files) + return image, err +} diff --git a/karapace/compatibility/__init__.py b/karapace/compatibility/__init__.py index e5f61e710..daa7edb8e 100644 --- a/karapace/compatibility/__init__.py +++ b/karapace/compatibility/__init__.py @@ -64,14 +64,17 @@ def check_jsonschema_compatibility(reader: Draft7Validator, writer: Draft7Valida return jsonschema_compatibility(reader, writer) -def check_protobuf_compatibility(reader: ProtobufSchema, writer: ProtobufSchema) -> SchemaCompatibilityResult: - return check_protobuf_schema_compatibility(reader, writer) +def check_protobuf_compatibility( + reader: ProtobufSchema, writer: ProtobufSchema, use_protopace: bool = False +) -> SchemaCompatibilityResult: + return check_protobuf_schema_compatibility(reader, writer, use_protopace) def check_compatibility( old_schema: ParsedTypedSchema, new_schema: ValidatedTypedSchema, compatibility_mode: CompatibilityModes, + use_protopace: bool = False, ) -> SchemaCompatibilityResult: """Check that `old_schema` and `new_schema` are compatible under `compatibility_mode`.""" if compatibility_mode is CompatibilityModes.NONE: @@ -148,23 +151,27 @@ def check_compatibility( result = check_protobuf_compatibility( reader=new_schema.schema, writer=old_schema.schema, + use_protopace=use_protopace, ) elif compatibility_mode in {CompatibilityModes.FORWARD, CompatibilityModes.FORWARD_TRANSITIVE}: result = check_protobuf_compatibility( reader=old_schema.schema, writer=new_schema.schema, + use_protopace=use_protopace, ) elif compatibility_mode in {CompatibilityModes.FULL, CompatibilityModes.FULL_TRANSITIVE}: result = check_protobuf_compatibility( reader=new_schema.schema, writer=old_schema.schema, + use_protopace=use_protopace, ) result = merge( result, check_protobuf_compatibility( reader=old_schema.schema, writer=new_schema.schema, + use_protopace=use_protopace, ), ) diff --git a/karapace/compatibility/protobuf/checks.py b/karapace/compatibility/protobuf/checks.py index b463b72ce..04a599f00 100644 --- a/karapace/compatibility/protobuf/checks.py +++ b/karapace/compatibility/protobuf/checks.py @@ -4,10 +4,25 @@ """ from avro.compatibility import SchemaCompatibilityResult, SchemaCompatibilityType from karapace.protobuf.compare_result import CompareResult +from karapace.protobuf.protopace import check_compatibility, IncompatibleError from karapace.protobuf.schema import ProtobufSchema -def check_protobuf_schema_compatibility(reader: ProtobufSchema, writer: ProtobufSchema) -> SchemaCompatibilityResult: +def check_protobuf_schema_compatibility( + reader: ProtobufSchema, writer: ProtobufSchema, use_protopace: bool = False +) -> SchemaCompatibilityResult: + if use_protopace: + old_proto = writer.to_proto() + new_proto = reader.to_proto() + try: + check_compatibility(new_proto, old_proto) + except IncompatibleError as err: + return SchemaCompatibilityResult( + compatibility=SchemaCompatibilityType.incompatible, + messages={str(err)}, + ) + return SchemaCompatibilityResult(SchemaCompatibilityType.compatible) + result = CompareResult() writer.compare(reader, result) if result.is_compatible(): diff --git a/karapace/config.py b/karapace/config.py index 9212f6348..dd3977e53 100644 --- a/karapace/config.py +++ b/karapace/config.py @@ -82,6 +82,8 @@ class Config(TypedDict): sentry: NotRequired[Mapping[str, object]] tags: NotRequired[Mapping[str, object]] + use_protopace: bool + class ConfigDefaults(Config, total=False): ... @@ -150,6 +152,7 @@ class ConfigDefaults(Config, total=False): "name_strategy_validation": True, "master_election_strategy": "lowest", "protobuf_runtime_directory": "runtime", + "use_protopace": True, } SECRET_CONFIG_OPTIONS = [SASL_PLAIN_PASSWORD] diff --git a/karapace/protobuf/protopace/__init__.py b/karapace/protobuf/protopace/__init__.py new file mode 100644 index 000000000..ec6c72f0b --- /dev/null +++ b/karapace/protobuf/protopace/__init__.py @@ -0,0 +1,6 @@ +""" +Copyright (c) 2024 Aiven Ltd +See LICENSE for details +""" + +from .protopace import check_compatibility, format_proto, IncompatibleError, Proto # noqa: F401 diff --git a/karapace/protobuf/protopace/bin/protopace-darwin-amd64.h b/karapace/protobuf/protopace/bin/protopace-darwin-amd64.h new file mode 100644 index 000000000..884226038 --- /dev/null +++ b/karapace/protobuf/protopace/bin/protopace-darwin-amd64.h @@ -0,0 +1,94 @@ +/* Code generated by cmd/cgo; DO NOT EDIT. */ + +/* package github.com/Aiven-Open/karapace/go/protopace */ + + +#line 1 "cgo-builtin-export-prolog" + +#include + +#ifndef GO_CGO_EXPORT_PROLOGUE_H +#define GO_CGO_EXPORT_PROLOGUE_H + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef struct { const char *p; ptrdiff_t n; } _GoString_; +#endif + +#endif + +/* Start of preamble from import "C" comments. */ + + +#line 3 "main.go" + +#include + +struct result { + char* res; + char* err; +}; + + +#line 1 "cgo-generated-wrapper" + + +/* End of preamble from import "C" comments. */ + + +/* Start of boilerplate cgo prologue. */ +#line 1 "cgo-gcc-export-header-prolog" + +#ifndef GO_CGO_PROLOGUE_H +#define GO_CGO_PROLOGUE_H + +typedef signed char GoInt8; +typedef unsigned char GoUint8; +typedef short GoInt16; +typedef unsigned short GoUint16; +typedef int GoInt32; +typedef unsigned int GoUint32; +typedef long long GoInt64; +typedef unsigned long long GoUint64; +typedef GoInt64 GoInt; +typedef GoUint64 GoUint; +typedef size_t GoUintptr; +typedef float GoFloat32; +typedef double GoFloat64; +#ifdef _MSC_VER +#include +typedef _Fcomplex GoComplex64; +typedef _Dcomplex GoComplex128; +#else +typedef float _Complex GoComplex64; +typedef double _Complex GoComplex128; +#endif + +/* + static assertion to make sure the file is being used on architecture + at least with matching size of GoInt. +*/ +typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1]; + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef _GoString_ GoString; +#endif +typedef void *GoMap; +typedef void *GoChan; +typedef struct { void *t; void *v; } GoInterface; +typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; + +#endif + +/* End of boilerplate cgo prologue. */ + +#ifdef __cplusplus +extern "C" { +#endif + +extern void SayHello(char* name); +extern struct result* FormatSchema(char* cSchemaName, char* cSchema, char** cDependencyNames, char** cDependencies, int depsLenght); +extern char* CheckCompatibility(char* cSchemaName, char* cSchema, char** cDependencyNames, char** cDependencies, int depsLenght, char* cSchemaNamePrev, char* cSchemaPrev, char** cDependencyNamesPrev, char** cDependenciesPrev, int depsLenghtPrev); + +#ifdef __cplusplus +} +#endif diff --git a/karapace/protobuf/protopace/bin/protopace-darwin-amd64.so b/karapace/protobuf/protopace/bin/protopace-darwin-amd64.so new file mode 100644 index 000000000..6b0c90dd0 Binary files /dev/null and b/karapace/protobuf/protopace/bin/protopace-darwin-amd64.so differ diff --git a/karapace/protobuf/protopace/bin/protopace-darwin-arm64.h b/karapace/protobuf/protopace/bin/protopace-darwin-arm64.h new file mode 100644 index 000000000..884226038 --- /dev/null +++ b/karapace/protobuf/protopace/bin/protopace-darwin-arm64.h @@ -0,0 +1,94 @@ +/* Code generated by cmd/cgo; DO NOT EDIT. */ + +/* package github.com/Aiven-Open/karapace/go/protopace */ + + +#line 1 "cgo-builtin-export-prolog" + +#include + +#ifndef GO_CGO_EXPORT_PROLOGUE_H +#define GO_CGO_EXPORT_PROLOGUE_H + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef struct { const char *p; ptrdiff_t n; } _GoString_; +#endif + +#endif + +/* Start of preamble from import "C" comments. */ + + +#line 3 "main.go" + +#include + +struct result { + char* res; + char* err; +}; + + +#line 1 "cgo-generated-wrapper" + + +/* End of preamble from import "C" comments. */ + + +/* Start of boilerplate cgo prologue. */ +#line 1 "cgo-gcc-export-header-prolog" + +#ifndef GO_CGO_PROLOGUE_H +#define GO_CGO_PROLOGUE_H + +typedef signed char GoInt8; +typedef unsigned char GoUint8; +typedef short GoInt16; +typedef unsigned short GoUint16; +typedef int GoInt32; +typedef unsigned int GoUint32; +typedef long long GoInt64; +typedef unsigned long long GoUint64; +typedef GoInt64 GoInt; +typedef GoUint64 GoUint; +typedef size_t GoUintptr; +typedef float GoFloat32; +typedef double GoFloat64; +#ifdef _MSC_VER +#include +typedef _Fcomplex GoComplex64; +typedef _Dcomplex GoComplex128; +#else +typedef float _Complex GoComplex64; +typedef double _Complex GoComplex128; +#endif + +/* + static assertion to make sure the file is being used on architecture + at least with matching size of GoInt. +*/ +typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1]; + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef _GoString_ GoString; +#endif +typedef void *GoMap; +typedef void *GoChan; +typedef struct { void *t; void *v; } GoInterface; +typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; + +#endif + +/* End of boilerplate cgo prologue. */ + +#ifdef __cplusplus +extern "C" { +#endif + +extern void SayHello(char* name); +extern struct result* FormatSchema(char* cSchemaName, char* cSchema, char** cDependencyNames, char** cDependencies, int depsLenght); +extern char* CheckCompatibility(char* cSchemaName, char* cSchema, char** cDependencyNames, char** cDependencies, int depsLenght, char* cSchemaNamePrev, char* cSchemaPrev, char** cDependencyNamesPrev, char** cDependenciesPrev, int depsLenghtPrev); + +#ifdef __cplusplus +} +#endif diff --git a/karapace/protobuf/protopace/bin/protopace-darwin-arm64.so b/karapace/protobuf/protopace/bin/protopace-darwin-arm64.so new file mode 100644 index 000000000..dde377d4f Binary files /dev/null and b/karapace/protobuf/protopace/bin/protopace-darwin-arm64.so differ diff --git a/karapace/protobuf/protopace/bin/protopace-linux-amd64.h b/karapace/protobuf/protopace/bin/protopace-linux-amd64.h new file mode 100644 index 000000000..884226038 --- /dev/null +++ b/karapace/protobuf/protopace/bin/protopace-linux-amd64.h @@ -0,0 +1,94 @@ +/* Code generated by cmd/cgo; DO NOT EDIT. */ + +/* package github.com/Aiven-Open/karapace/go/protopace */ + + +#line 1 "cgo-builtin-export-prolog" + +#include + +#ifndef GO_CGO_EXPORT_PROLOGUE_H +#define GO_CGO_EXPORT_PROLOGUE_H + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef struct { const char *p; ptrdiff_t n; } _GoString_; +#endif + +#endif + +/* Start of preamble from import "C" comments. */ + + +#line 3 "main.go" + +#include + +struct result { + char* res; + char* err; +}; + + +#line 1 "cgo-generated-wrapper" + + +/* End of preamble from import "C" comments. */ + + +/* Start of boilerplate cgo prologue. */ +#line 1 "cgo-gcc-export-header-prolog" + +#ifndef GO_CGO_PROLOGUE_H +#define GO_CGO_PROLOGUE_H + +typedef signed char GoInt8; +typedef unsigned char GoUint8; +typedef short GoInt16; +typedef unsigned short GoUint16; +typedef int GoInt32; +typedef unsigned int GoUint32; +typedef long long GoInt64; +typedef unsigned long long GoUint64; +typedef GoInt64 GoInt; +typedef GoUint64 GoUint; +typedef size_t GoUintptr; +typedef float GoFloat32; +typedef double GoFloat64; +#ifdef _MSC_VER +#include +typedef _Fcomplex GoComplex64; +typedef _Dcomplex GoComplex128; +#else +typedef float _Complex GoComplex64; +typedef double _Complex GoComplex128; +#endif + +/* + static assertion to make sure the file is being used on architecture + at least with matching size of GoInt. +*/ +typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1]; + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef _GoString_ GoString; +#endif +typedef void *GoMap; +typedef void *GoChan; +typedef struct { void *t; void *v; } GoInterface; +typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; + +#endif + +/* End of boilerplate cgo prologue. */ + +#ifdef __cplusplus +extern "C" { +#endif + +extern void SayHello(char* name); +extern struct result* FormatSchema(char* cSchemaName, char* cSchema, char** cDependencyNames, char** cDependencies, int depsLenght); +extern char* CheckCompatibility(char* cSchemaName, char* cSchema, char** cDependencyNames, char** cDependencies, int depsLenght, char* cSchemaNamePrev, char* cSchemaPrev, char** cDependencyNamesPrev, char** cDependenciesPrev, int depsLenghtPrev); + +#ifdef __cplusplus +} +#endif diff --git a/karapace/protobuf/protopace/bin/protopace-linux-amd64.so b/karapace/protobuf/protopace/bin/protopace-linux-amd64.so new file mode 100644 index 000000000..cd94c59d3 Binary files /dev/null and b/karapace/protobuf/protopace/bin/protopace-linux-amd64.so differ diff --git a/karapace/protobuf/protopace/bin/protopace-linux-arm64.h b/karapace/protobuf/protopace/bin/protopace-linux-arm64.h new file mode 100644 index 000000000..884226038 --- /dev/null +++ b/karapace/protobuf/protopace/bin/protopace-linux-arm64.h @@ -0,0 +1,94 @@ +/* Code generated by cmd/cgo; DO NOT EDIT. */ + +/* package github.com/Aiven-Open/karapace/go/protopace */ + + +#line 1 "cgo-builtin-export-prolog" + +#include + +#ifndef GO_CGO_EXPORT_PROLOGUE_H +#define GO_CGO_EXPORT_PROLOGUE_H + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef struct { const char *p; ptrdiff_t n; } _GoString_; +#endif + +#endif + +/* Start of preamble from import "C" comments. */ + + +#line 3 "main.go" + +#include + +struct result { + char* res; + char* err; +}; + + +#line 1 "cgo-generated-wrapper" + + +/* End of preamble from import "C" comments. */ + + +/* Start of boilerplate cgo prologue. */ +#line 1 "cgo-gcc-export-header-prolog" + +#ifndef GO_CGO_PROLOGUE_H +#define GO_CGO_PROLOGUE_H + +typedef signed char GoInt8; +typedef unsigned char GoUint8; +typedef short GoInt16; +typedef unsigned short GoUint16; +typedef int GoInt32; +typedef unsigned int GoUint32; +typedef long long GoInt64; +typedef unsigned long long GoUint64; +typedef GoInt64 GoInt; +typedef GoUint64 GoUint; +typedef size_t GoUintptr; +typedef float GoFloat32; +typedef double GoFloat64; +#ifdef _MSC_VER +#include +typedef _Fcomplex GoComplex64; +typedef _Dcomplex GoComplex128; +#else +typedef float _Complex GoComplex64; +typedef double _Complex GoComplex128; +#endif + +/* + static assertion to make sure the file is being used on architecture + at least with matching size of GoInt. +*/ +typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1]; + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef _GoString_ GoString; +#endif +typedef void *GoMap; +typedef void *GoChan; +typedef struct { void *t; void *v; } GoInterface; +typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; + +#endif + +/* End of boilerplate cgo prologue. */ + +#ifdef __cplusplus +extern "C" { +#endif + +extern void SayHello(char* name); +extern struct result* FormatSchema(char* cSchemaName, char* cSchema, char** cDependencyNames, char** cDependencies, int depsLenght); +extern char* CheckCompatibility(char* cSchemaName, char* cSchema, char** cDependencyNames, char** cDependencies, int depsLenght, char* cSchemaNamePrev, char* cSchemaPrev, char** cDependencyNamesPrev, char** cDependenciesPrev, int depsLenghtPrev); + +#ifdef __cplusplus +} +#endif diff --git a/karapace/protobuf/protopace/bin/protopace-linux-arm64.so b/karapace/protobuf/protopace/bin/protopace-linux-arm64.so new file mode 100644 index 000000000..2d29b1b41 Binary files /dev/null and b/karapace/protobuf/protopace/bin/protopace-linux-arm64.so differ diff --git a/karapace/protobuf/protopace/protopace.py b/karapace/protobuf/protopace/protopace.py new file mode 100644 index 000000000..eb6b9d209 --- /dev/null +++ b/karapace/protobuf/protopace/protopace.py @@ -0,0 +1,238 @@ +""" +Copyright (c) 2024 Aiven Ltd +See LICENSE for details +""" + +from dataclasses import dataclass, field +from functools import cached_property +from karapace.errors import InvalidSchema +from typing import List + +import ctypes +import os +import platform +import timeit + +system = platform.system().lower() +arch = platform.machine().lower() +file_dir = os.path.dirname(os.path.abspath(__file__)) + +if arch == "x86_64": + arch = "amd64" +elif arch == "aarch64": + arch = "arm64" + +lib_file = os.path.join(file_dir, f"bin/protopace-{system}-{arch}.so") + +if not os.path.exists(lib_file): + raise RuntimeError(f"Unsupported platform: {system}-{arch}") + +lib = ctypes.CDLL(lib_file) + +lib.FormatSchema.restype = ctypes.c_void_p +lib.CheckCompatibility.restype = ctypes.c_char_p + + +class FormatResult(ctypes.Structure): + _fields_ = [ + ("res", ctypes.c_char_p), + ("err", ctypes.c_char_p), + ] + + +@dataclass +class Proto: + name: str + schema: str + dependencies: List["Proto"] = field(default_factory=list) + + @cached_property + def all_dependencies(self) -> List["Proto"]: + dependencies = {} + for dep in self.dependencies: + if dep.dependencies: + dependencies.update([(d.name, d) for d in dep.all_dependencies]) + dependencies[dep.name] = dep + return list(dependencies.values()) + + +class IncompatibleError(Exception): + pass + + +def format_proto(proto: Proto) -> str: + length = len(proto.all_dependencies) + c_dependencies = (ctypes.c_char_p * length)(*[d.schema.encode() for d in proto.all_dependencies]) + c_dependency_names = (ctypes.c_char_p * length)(*[d.name.encode() for d in proto.all_dependencies]) + + res_ptr = lib.FormatSchema(proto.name.encode(), proto.schema.encode(), c_dependency_names, c_dependencies, length) + res = FormatResult.from_address(res_ptr) + + if res.err: + err = res.err + msg = err.decode() + lib.free(ctypes.c_void_p(res_ptr)) + raise InvalidSchema(msg) + + result = res.res.decode() + lib.free(ctypes.c_void_p(res_ptr)) + return result + + +def check_compatibility(proto: Proto, prev_proto: Proto) -> None: + length = len(proto.all_dependencies) + c_dependencies = (ctypes.c_char_p * length)(*[d.schema.encode() for d in proto.all_dependencies]) + c_dependency_names = (ctypes.c_char_p * length)(*[d.name.encode() for d in proto.all_dependencies]) + + prev_length = len(prev_proto.all_dependencies) + prev_c_dependencies = (ctypes.c_char_p * prev_length)(*[d.schema.encode() for d in prev_proto.all_dependencies]) + prev_c_dependency_names = (ctypes.c_char_p * prev_length)(*[d.name.encode() for d in prev_proto.all_dependencies]) + + err = lib.CheckCompatibility( + proto.name.encode(), + proto.schema.encode(), + c_dependency_names, + c_dependencies, + length, + prev_proto.name.encode(), + prev_proto.schema.encode(), + prev_c_dependency_names, + prev_c_dependencies, + prev_length, + ) + + if err is not None: + msg = err.decode() + raise IncompatibleError(msg) + + +SCHEMA = """ +syntax = "proto3"; + +package my.awesome.customer.v1; + +import "my/awesome/customer/v1/nested_value.proto"; +import "google/protobuf/timestamp.proto"; + +option ruby_package = "My::Awesome::Customer::V1"; +option csharp_namespace = "my.awesome.customer.V1"; +option go_package = "github.com/customer/api/my/awesome/customer/v1;dspv1"; +option java_multiple_files = true; +option java_outer_classname = "EventValueProto"; +option java_package = "com.my.awesome.customer.v1"; +option objc_class_prefix = "TDD"; + +message Local { + message NestedValue { + string foo = 1; + } +} + +message EventValue { + NestedValue nested_value = 1; + google.protobuf.Timestamp created_at = 2; + Status status = 3; + Local.NestedValue local_nested_value = 4; +} +""" + +PREV_SCHEMA = """ +syntax = "proto3"; + +package my.awesome.customer.v1; + +import "my/awesome/customer/v1/nested_value.proto"; +import "google/protobuf/timestamp.proto"; + +option ruby_package = "My::Awesome::Customer::V1"; +option csharp_namespace = "my.awesome.customer.V1"; +option go_package = "github.com/customer/api/my/awesome/customer/v1;dspv1"; +option java_multiple_files = true; +option java_outer_classname = "EventValueProto"; +option java_package = "com.my.awesome.customer.v1"; +option objc_class_prefix = "TDD"; + +message Local { + message NestedValue { + string foo = 1; + } +} + +message EventValue { + NestedValue nested_value = 1; + google.protobuf.Timestamp created_at = 2; + Status status = 3; + Local.NestedValue local_nested_value = 5; +} +""" + +DEPENDENCY = """ +syntax = "proto3"; +package my.awesome.customer.v1; + +message NestedValue { + string value = 1; +} + +enum Status { + UNKNOWN = 0; + ACTIVE = 1; + INACTIVE = 2; +} +""" + + +def _format_time(seconds: float) -> str: + units = [("s", 1), ("ms", 1e-3), ("µs", 1e-6), ("ns", 1e-9)] + for unit, factor in units: + if seconds >= factor: + return f"{seconds / factor:.3f} {unit}" + return f"{seconds:.3f} s" + + +def _test_format() -> None: + proto = Proto("test.proto", SCHEMA, [Proto("my/awesome/customer/v1/nested_value.proto", DEPENDENCY)]) + res = format_proto(proto) + print(res) + + +def _test_check_compatibility() -> None: + proto = Proto("test.proto", SCHEMA, [Proto("my/awesome/customer/v1/nested_value.proto", DEPENDENCY)]) + prev_proto = Proto("test.proto", PREV_SCHEMA, [Proto("my/awesome/customer/v1/nested_value.proto", DEPENDENCY)]) + try: + check_compatibility(proto, prev_proto) + except IncompatibleError as err: + print(err) + + +def _time_format() -> None: + def test() -> None: + proto = Proto("test.proto", SCHEMA, [Proto("my/awesome/customer/v1/nested_value.proto", DEPENDENCY)]) + format_proto(proto) + + number = 10000 + seconds = timeit.timeit(test, number=number) + + print("----- Format -----") + print(f"Total time: {_format_time(seconds)}") + print(f"Execution time per loop: {_format_time(seconds / number)}") + + +def _time_check_compatibility() -> None: + def test() -> None: + proto = Proto("test.proto", SCHEMA, [Proto("my/awesome/customer/v1/nested_value.proto", DEPENDENCY)]) + check_compatibility(proto, proto) + + number = 10000 + seconds = timeit.timeit(test, number=number) + + print("----- Compatibility Check -----") + print(f"Total time: {_format_time(seconds)}") + print(f"Execution time per loop: {_format_time(seconds / number)}") + + +if __name__ == "__main__": + _test_format() + _test_check_compatibility() + _time_format() + _time_check_compatibility() diff --git a/karapace/protobuf/schema.py b/karapace/protobuf/schema.py index 9c407bdcd..e15d070fb 100644 --- a/karapace/protobuf/schema.py +++ b/karapace/protobuf/schema.py @@ -21,6 +21,7 @@ from karapace.protobuf.option_element import OptionElement from karapace.protobuf.proto_file_element import ProtoFileElement from karapace.protobuf.proto_parser import ProtoParser +from karapace.protobuf.protopace import Proto from karapace.protobuf.serialization import deserialize, serialize from karapace.protobuf.type_element import TypeElement from karapace.protobuf.utils import append_documentation, append_indented @@ -267,10 +268,17 @@ def __init__( self.proto_file_element = deserialize(schema) except binascii.Error: # If not base64 formatted self.proto_file_element = ProtoParser.parse(DEFAULT_LOCATION, schema) - + self._schema_str = schema self.references = references self.dependencies = dependencies + def to_proto(self, name="noname.proto") -> Proto: + dependencies = [] + if self.dependencies: + for _, dep in self.dependencies.items(): + dependencies.append(dep.get_schema().to_proto(name=dep.name)) + return Proto(name, self._schema_str, dependencies) + def type_in_tree(self, tree: TypeTree, remaining_tokens: list[str]) -> TypeTree | None: if remaining_tokens: to_seek = remaining_tokens.pop() diff --git a/karapace/schema_models.py b/karapace/schema_models.py index d21917025..9f092ec5b 100644 --- a/karapace/schema_models.py +++ b/karapace/schema_models.py @@ -20,12 +20,13 @@ SchemaParseException as ProtobufSchemaParseException, ) from karapace.protobuf.proto_normalizations import NormalizedProtobufSchema +from karapace.protobuf.protopace import format_proto, Proto from karapace.protobuf.schema import ProtobufSchema from karapace.schema_references import Reference from karapace.schema_type import SchemaType from karapace.typing import JsonObject, SchemaId, Subject, Version, VersionTag from karapace.utils import assert_never, json_decode, json_encode, JSONDecodeError -from typing import Any, cast, Dict, Final, final, Mapping, Sequence +from typing import Any, cast, Dict, Final, Mapping, Sequence import hashlib import logging @@ -93,6 +94,7 @@ def __init__( schema: Draft7Validator | AvroSchema | ProtobufSchema | None = None, references: Sequence[Reference] | None = None, dependencies: Mapping[str, Dependency] | None = None, + use_protopace: bool = False, ) -> None: """Schema with type information @@ -105,9 +107,12 @@ def __init__( self.schema_type: Final = schema_type self.references: Final = references self.dependencies: Final = dependencies - self.schema_str: Final = TypedSchema.normalize_schema_str(schema_str, schema_type, schema) + self.schema_str: Final = schema_str self.max_id: SchemaId | None = None self._fingerprint_cached: str | None = None + self.use_protopace = use_protopace + self._schema = schema + self.normalize_schema_str() def to_dict(self) -> JsonObject: if self.schema_type is SchemaType.PROTOBUF: @@ -123,34 +128,37 @@ def fingerprint(self) -> str: self._fingerprint_cached = hashlib.sha1(fingerprint_str.encode("utf8")).hexdigest() return self._fingerprint_cached - # This is marked @final because __init__ references this statically, hence - # allowing overriding this in a subclass could lead to confusing bugs. - @staticmethod - @final - def normalize_schema_str( - schema_str: str, - schema_type: SchemaType, - schema: Draft7Validator | AvroSchema | ProtobufSchema | None = None, - ) -> str: - if schema_type is SchemaType.AVRO or schema_type is SchemaType.JSONSCHEMA: + def to_proto(self, name="noname.proto") -> Proto: + if self.schema_type is not SchemaType.PROTOBUF: + raise InvalidSchema("Only supported for Protobuf") + + dependencies = [] + if self.dependencies: + for _, dep in self.dependencies.items(): + dependencies.append(dep.get_schema().to_proto(name=dep.name)) + return Proto(name, self.schema_str, dependencies) + + def normalize_schema_str(self) -> None: + if self.use_protopace and self.schema_type is SchemaType.PROTOBUF: + self.schema_str = format_proto(self.to_proto()) + if self.schema_type is SchemaType.AVRO or self.schema_type is SchemaType.JSONSCHEMA: try: - schema_str = json_encode(json_decode(schema_str), compact=True, sort_keys=True) + self.schema_str = json_encode(json_decode(self.schema_str), compact=True, sort_keys=True) except JSONDecodeError as e: LOG.info("Schema is not valid JSON") raise e - elif schema_type == SchemaType.PROTOBUF: - if schema: - schema_str = str(schema) + elif self.schema_type == SchemaType.PROTOBUF: + if self._schema: + self.schema_str = str(self._schema) else: try: - schema_str = str(parse_protobuf_schema_definition(schema_str, None, None, False)) + self.schema_strschema_str = str(parse_protobuf_schema_definition(self.schema_str, None, None, False)) except InvalidSchema as e: LOG.info("Schema is not valid ProtoBuf definition") raise e else: - assert_never(schema_type) - return schema_str + assert_never(self.schema_type) def __str__(self) -> str: return self.schema_str @@ -188,6 +196,7 @@ def parse( references: Sequence[Reference] | None = None, dependencies: Mapping[str, Dependency] | None = None, normalize: bool = False, + use_protopace: bool = False, ) -> ParsedTypedSchema: if schema_type not in [SchemaType.AVRO, SchemaType.JSONSCHEMA, SchemaType.PROTOBUF]: raise InvalidSchema(f"Unknown parser {schema_type} for {schema_str}") @@ -235,6 +244,7 @@ def parse( schema=parsed_schema, references=references, dependencies=dependencies, + use_protopace=use_protopace, ) @@ -264,6 +274,7 @@ def __init__( schema: Draft7Validator | AvroSchema | ProtobufSchema, references: Sequence[Reference] | None = None, dependencies: Mapping[str, Dependency] | None = None, + use_protopace: bool = False, ) -> None: self._schema_cached: Draft7Validator | AvroSchema | ProtobufSchema | None = schema @@ -273,6 +284,7 @@ def __init__( references=references, dependencies=dependencies, schema=schema, + use_protopace=use_protopace, ) @staticmethod @@ -282,6 +294,7 @@ def parse( references: Sequence[Reference] | None = None, dependencies: Mapping[str, Dependency] | None = None, normalize: bool = False, + use_protopace: bool = False, ) -> ParsedTypedSchema: return parse( schema_type=schema_type, @@ -291,6 +304,7 @@ def parse( references=references, dependencies=dependencies, normalize=normalize, + use_protopace=use_protopace, ) def __str__(self) -> str: @@ -350,6 +364,7 @@ def __init__( schema: Draft7Validator | AvroSchema | ProtobufSchema, references: list[Reference] | None = None, dependencies: dict[str, Dependency] | None = None, + use_protopace: bool = False, ) -> None: super().__init__( schema_type=schema_type, @@ -357,6 +372,7 @@ def __init__( references=references, dependencies=dependencies, schema=schema, + use_protopace=use_protopace, ) @staticmethod @@ -366,6 +382,7 @@ def parse( references: Sequence[Reference] | None = None, dependencies: Mapping[str, Dependency] | None = None, normalize: bool = False, + use_protopace: bool = False, ) -> ValidatedTypedSchema: parsed_schema = parse( schema_type=schema_type, @@ -375,6 +392,7 @@ def parse( references=references, dependencies=dependencies, normalize=normalize, + use_protopace=use_protopace, ) return cast(ValidatedTypedSchema, parsed_schema) diff --git a/karapace/schema_reader.py b/karapace/schema_reader.py index 493019233..21c310247 100644 --- a/karapace/schema_reader.py +++ b/karapace/schema_reader.py @@ -133,6 +133,7 @@ def __init__( self.timeout_s = MESSAGE_CONSUME_TIMEOUT_SECONDS self.max_messages_to_process = MAX_MESSAGES_TO_CONSUME_ON_STARTUP self.config = config + self.use_protopace = config["use_protopace"] self.database = database self.admin_client: KafkaAdminClient | None = None @@ -544,6 +545,7 @@ def _handle_msg_schema(self, key: dict, value: dict | None) -> None: normalize=False, ) schema_str = str(parsed_schema) + except InvalidSchema: LOG.exception("Schema is not valid ProtoBuf definition") return @@ -558,6 +560,7 @@ def _handle_msg_schema(self, key: dict, value: dict | None) -> None: references=resolved_references, dependencies=resolved_dependencies, schema=parsed_schema, + use_protopace=self.use_protopace, ) except (InvalidSchema, JSONDecodeError): return diff --git a/karapace/schema_registry_apis.py b/karapace/schema_registry_apis.py index 4ef7b884a..7ebcdeb58 100644 --- a/karapace/schema_registry_apis.py +++ b/karapace/schema_registry_apis.py @@ -449,8 +449,10 @@ async def compatibility_check( old_schema=old_schema, new_schema=new_schema, compatibility_mode=compatibility_mode, + use_protopace=self.config["use_protopace"], ) if is_incompatible(result): + self.log.info("incompatible: %s", result.messages) self.r({"is_compatible": False}, content_type) self.r({"is_compatible": True}, content_type) @@ -1121,6 +1123,7 @@ async def subjects_schema_post( references=references, dependencies=new_schema_dependencies, normalize=normalize, + use_protopace=self.config["use_protopace"], ) except InvalidSchema: self.log.warning("Invalid schema: %r", schema_str) @@ -1153,6 +1156,7 @@ async def subjects_schema_post( references=other_references, dependencies=other_dependencies, normalize=normalize, + use_protopace=self.config["use_protopace"], ) except InvalidSchema as e: failed_schema_id = schema_version.schema_id @@ -1222,6 +1226,7 @@ async def subject_post( references=references, dependencies=resolved_dependencies, normalize=normalize, + use_protopace=self.config["use_protopace"], ) except (InvalidReferences, InvalidSchema, InvalidSchemaType) as e: self.log.warning("Invalid schema: %r", body["schema"], exc_info=True) diff --git a/setup.py b/setup.py index 3b50270d1..fa8f24d3b 100644 --- a/setup.py +++ b/setup.py @@ -70,4 +70,5 @@ "Topic :: Database :: Database Engines/Servers", "Topic :: Software Development :: Libraries", ], + include_package_data=True, ) diff --git a/tests/integration/test_dependencies_compatibility_protobuf.py b/tests/integration/test_dependencies_compatibility_protobuf.py index 2bacbdf7b..837cebf79 100644 --- a/tests/integration/test_dependencies_compatibility_protobuf.py +++ b/tests/integration/test_dependencies_compatibility_protobuf.py @@ -323,10 +323,10 @@ async def test_protobuf_schema_compatibility_dependencies1g(registry_async_clien evolved_schema = """ syntax = "proto3"; package a1; -import "google/type/postal_address.proto"; +import "google/protobuf/duration.proto"; message TestMessage { message V { - google.type.PostalAddress h = 1; + google.protobuf.Duration h = 1; int32 x = 2; } string t = 1; @@ -369,10 +369,10 @@ async def test_protobuf_schema_compatibility_dependencies1g_otherway(registry_as original_schema = """ syntax = "proto3"; package a1; -import "google/type/postal_address.proto"; +import "google/protobuf/duration.proto"; message TestMessage { message V { - google.type.PostalAddress h = 1; + google.protobuf.Duration h = 1; int32 x = 2; } string t = 1; @@ -617,13 +617,13 @@ async def test_protobuf_customer_update_when_having_references(registry_async_cl syntax = "proto3"; package a1; import "place.proto"; -import "google/type/postal_address.proto"; +import "google/protobuf/duration.proto"; // @producer: another comment message Customer { string name = 1; int32 code = 2; Place place = 3; - google.type.PostalAddress address = 4; + google.protobuf.Duration address = 4; } """ body = { @@ -645,13 +645,13 @@ async def test_protobuf_customer_update_when_having_references(registry_async_cl syntax = "proto3"; package a1; import "place.proto"; -import "google/type/postal_address.proto"; +import "google/protobuf/duration.proto"; // @consumer: the comment was incorrect, updating it now message Customer { string name = 1; int32 code = 2; Place place = 3; - google.type.PostalAddress address = 4; + google.protobuf.Duration address = 4; } """ diff --git a/tests/integration/test_schema_protobuf.py b/tests/integration/test_schema_protobuf.py index 10df70637..febbea601 100644 --- a/tests/integration/test_schema_protobuf.py +++ b/tests/integration/test_schema_protobuf.py @@ -126,7 +126,8 @@ async def test_protobuf_schema_normalization(registry_async_client: Client, trai |message TestMessage { | message Value { | string str2 = 1; - | Enu x = 2; + | int32 x = 2; + | Enu enu = 3; | } | string test = 1; | .a1.TestMessage.Value val = 2; @@ -236,7 +237,11 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None |syntax = "proto3"; |package a1; |import "Customer.proto"; + |import "google/protobuf/descriptor.proto"; |message TestMessage { + | extend google.protobuf.OneofOptions { + | bool my_option = 50002; + | } | enum Enum { | HIGH = 0; | MIDDLE = 1; @@ -404,7 +409,11 @@ async def test_protobuf_schema_verifier(registry_async_client: Client) -> None: |syntax = "proto3"; |package a1; |import "Customer.proto"; + |import "google/protobuf/descriptor.proto"; |message TestMessage { + | extend google.protobuf.OneofOptions { + | bool my_option = 50002; + | } | enum Enum { | HIGH = 0; | MIDDLE = 1; diff --git a/tests/unit/protobuf/test_protobuf_compatibility.py b/tests/unit/protobuf/test_protobuf_compatibility.py index eaf9b98a5..eb3df5542 100644 --- a/tests/unit/protobuf/test_protobuf_compatibility.py +++ b/tests/unit/protobuf/test_protobuf_compatibility.py @@ -7,6 +7,9 @@ from karapace.protobuf.location import Location from karapace.protobuf.proto_file_element import ProtoFileElement from karapace.protobuf.proto_parser import ProtoParser +from karapace.protobuf.protopace import check_compatibility, Proto + +import pytest location: Location = Location("some/folder", "file.proto") @@ -45,6 +48,39 @@ def test_compatibility_package(): assert result.is_compatible() +def test_compatibility_package_with_protopace(): + self_schema = """ + |syntax = "proto3"; + |package a1; + |message TestMessage { + | message Value { + | string str = 1; + | } + | string test = 1; + | .a1.TestMessage.Value val = 2; + |} + |""" + + other_schema = """ + |syntax = "proto3"; + |package a2; + |message TestMessage { + | message Value { + | string str = 1; + | } + | string test = 1; + | .a2.TestMessage.Value val = 2; + |} + |""" + self_schema = trim_margin(self_schema) + other_schema = trim_margin(other_schema) + + proto = Proto("test.proto", other_schema) + prev_proto = Proto("test.proto", self_schema) + + check_compatibility(proto, prev_proto) + + def test_compatibility_field_add(): self_schema = """ |syntax = "proto3"; @@ -80,6 +116,39 @@ def test_compatibility_field_add(): assert result.is_compatible() +def test_compatibility_field_add_with_protopace(): + self_schema = """ + |syntax = "proto3"; + |package a1; + |message TestMessage { + | message Value { + | string str = 1; + | } + | string test = 1; + | .a1.TestMessage.Value val = 2; + |} + |""" + + other_schema = """ + |syntax = "proto3"; + |package a1; + |message TestMessage { + | message Value { + | string str = 1; + | string str2 = 2; + | } + | string test = 1; + | .a1.TestMessage.Value val = 2; + |} + |""" + + self_schema = trim_margin(self_schema) + other_schema = trim_margin(other_schema) + proto = Proto("test.proto", other_schema) + prev_proto = Proto("test.proto", self_schema) + check_compatibility(proto, prev_proto) + + def test_compatibility_field_drop(): self_schema = """ |syntax = "proto3"; @@ -115,6 +184,41 @@ def test_compatibility_field_drop(): assert result.is_compatible() +def test_compatibility_field_drop_with_protopace(): + self_schema = """ + |syntax = "proto3"; + |package a1; + |message TestMessage { + | message Value { + | string str = 1; + | string str2 = 2; + | } + | string test = 1; + | .a1.TestMessage.Value val = 2; + |} + |""" + + other_schema = """ + |syntax = "proto3"; + |package a1; + |message TestMessage { + | message Value { + | string str = 1; + | } + | string test = 1; + | .a1.TestMessage.Value val = 2; + |} + |""" + + self_schema = trim_margin(self_schema) + other_schema = trim_margin(other_schema) + + proto = Proto("test.proto", other_schema) + prev_proto = Proto("test.proto", self_schema) + + check_compatibility(proto, prev_proto) + + def test_compatibility_field_add_drop(): self_schema = """ |syntax = "proto3"; @@ -149,6 +253,40 @@ def test_compatibility_field_add_drop(): assert result.is_compatible() +def test_compatibility_field_add_drop_with_protopace(): + self_schema = """ + |syntax = "proto3"; + |package a1; + |message TestMessage { + | message Value { + | string str2 = 1; + | } + | string test = 1; + | .a1.TestMessage.Value val = 2; + |} + |""" + + other_schema = """ + |syntax = "proto3"; + |package a1; + |message TestMessage { + | message Value { + | string str = 1; + | } + | string test = 1; + | .a1.TestMessage.Value val = 2; + |} + |""" + + self_schema = trim_margin(self_schema) + other_schema = trim_margin(other_schema) + + proto = Proto("test.proto", other_schema) + prev_proto = Proto("test.proto", self_schema) + + check_compatibility(proto, prev_proto) + + def test_compatibility_enum_add(): self_schema = """ |syntax = "proto3"; @@ -190,6 +328,49 @@ def test_compatibility_enum_add(): assert result.is_compatible() +def test_compatibility_enum_add_with_protopace(): + self_schema = """ + |syntax = "proto3"; + |package a1; + |message TestMessage { + | message Value { + | string str2 = 1; + | int32 x = 2; + | } + | string test = 1; + | .a1.TestMessage.Value val = 2; + |} + |""" + + other_schema = """ + |syntax = "proto3"; + |package a1; + |message TestMessage { + | message Value { + | string str2 = 1; + | Enu x = 2; + | } + | string test = 1; + | .a1.TestMessage.Value val = 2; + | enum Enu { + | A = 0; + | B = 1; + | } + |} + |""" + + self_schema = trim_margin(self_schema) + other_schema = trim_margin(other_schema) + + proto = Proto("test.proto", other_schema) + prev_proto = Proto("test.proto", self_schema) + + # Note: This will be interpreted as a type change and will be regognized as breaking with vanilla buf + with pytest.raises(Exception) as e: + check_compatibility(proto, prev_proto) + assert 'Field "2" with name "x" on message "Value" changed type from "int32" to "enum".' in str(e) + + def test_compatibility_ordering_change_msg(): self_schema = """\ syntax = "proto3"; @@ -226,6 +407,39 @@ def test_compatibility_ordering_change_msg(): assert len(result.result) == 0 +def test_compatibility_ordering_change_msg_with_protopace(): + self_schema = """\ +syntax = "proto3"; +package tc4; + +message Fred { + int32 fredfield = 1; +} + +message HodoCode { + int32 hodofield = 1; +} +""" + + other_schema = """\ +syntax = "proto3"; +package tc4; + +message HodoCode { + int32 hodofield = 1; +} + +message Fred { + int32 fredfield = 1; +} +""" + # Note: tag number must be > 0. Had to be changed from original test. + proto = Proto("test.proto", other_schema) + prev_proto = Proto("test.proto", self_schema) + + check_compatibility(proto, prev_proto) + + def test_compatibility_ordering_change(): self_schema = """\ syntax = "proto3"; @@ -264,6 +478,40 @@ def test_compatibility_ordering_change(): assert result.result[0].modification == Modification.FIELD_ADD +def test_compatibility_ordering_change_with_protopace(): + self_schema = """\ +syntax = "proto3"; +package tc4; + +message Fred { + HodoCode hodecode = 1; +} + +enum HodoCode { + HODO_CODE_UNSPECIFIED = 0; +} +""" + + other_schema = """\ +syntax = "proto3"; +package tc4; + +enum HodoCode { + HODO_CODE_UNSPECIFIED = 0; +} + +message Fred { + HodoCode hodecode = 1; + string id = 2; +} +""" + + proto = Proto("test.proto", other_schema) + prev_proto = Proto("test.proto", self_schema) + + check_compatibility(proto, prev_proto) + + def test_compatibility_ordering_change2(): self_schema = """\ syntax = "proto3"; @@ -302,6 +550,40 @@ def test_compatibility_ordering_change2(): assert result.result[0].modification == Modification.FIELD_ADD +def test_compatibility_ordering_change2_with_protopace(): + self_schema = """\ +syntax = "proto3"; +package tc4; + +message Fred { + HodoCode hodecode = 1; +} + +enum HodoCode { + HODO_CODE_UNSPECIFIED = 0; +} +""" + + other_schema = """\ +syntax = "proto3"; +package tc4; + +message Fred { + HodoCode hodecode = 1; + string id = 2; +} + +enum HodoCode { + HODO_CODE_UNSPECIFIED = 0; +} +""" + + proto = Proto("test.proto", other_schema) + prev_proto = Proto("test.proto", self_schema) + + check_compatibility(proto, prev_proto) + + def test_compatibility_field_tag_change(): self_schema = """\ syntax = "proto3"; @@ -336,3 +618,33 @@ def test_compatibility_field_tag_change(): (Modification.FIELD_DROP, "Foo.4"), (Modification.FIELD_ADD, "Foo.5"), } + + +def test_compatibility_field_tag_change_with_protopace(): + self_schema = """\ +syntax = "proto3"; +package pkg; +message Foo { + string fieldA = 1; + string fieldB = 2; + string fieldC = 3; + string fieldX = 4; +} +""" + + other_schema = """\ +syntax = "proto3"; +package pkg; +message Foo { + string fieldA = 1; + string fieldB = 2; + string fieldC = 3; + string fieldX = 5; +} +""" + + proto = Proto("test.proto", other_schema) + prev_proto = Proto("test.proto", self_schema) + + # Note: because we allow field name change this is not recognized as incompatible + check_compatibility(proto, prev_proto) diff --git a/tests/unit/protobuf/test_protobuf_normalization.py b/tests/unit/protobuf/test_protobuf_normalization.py index b772b293c..d2d132cf8 100644 --- a/tests/unit/protobuf/test_protobuf_normalization.py +++ b/tests/unit/protobuf/test_protobuf_normalization.py @@ -6,6 +6,7 @@ from karapace.protobuf.location import Location from karapace.protobuf.proto_normalizations import normalize from karapace.protobuf.proto_parser import ProtoParser +from karapace.protobuf.protopace import check_compatibility, format_proto, Proto import pytest @@ -63,10 +64,20 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { option (my_option) = "my_value"; option (my_option2) = "my_value2"; option (my_option3) = "my_value3"; + + ACTIVE = 0; } """ @@ -75,10 +86,20 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { option (my_option3) = "my_value3"; option (my_option) = "my_value"; option (my_option2) = "my_value2"; + + ACTIVE = 0; } """ @@ -87,6 +108,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.ServiceOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + service MyService { option (my_option) = "my_value"; option (my_option2) = "my_value2"; @@ -99,6 +128,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.ServiceOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + service MyService { option (my_option3) = "my_value3"; option (my_option) = "my_value"; @@ -111,6 +148,18 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.MethodOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + +message Foo { + string res = 1; +} + service MyService { rpc MyRpc (Foo) returns (Foo) { option (my_option) = "my_value"; @@ -125,6 +174,18 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.MethodOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + +message Foo { + string res = 1; +} + service MyService { rpc MyRpc (Foo) returns (Foo) { option (my_option3) = "my_value3"; @@ -139,6 +200,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.MessageOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { string fieldA = 1; } @@ -155,6 +224,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.MessageOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { string fieldA = 1; } @@ -171,11 +248,21 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.OneofOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { oneof my_oneof { option (my_option) = "my_value"; option (my_option2) = "my_value2"; option (my_option3) = "my_value3"; + + string test = 1; } } """ @@ -185,11 +272,21 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.OneofOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { oneof my_oneof { option (my_option3) = "my_value3"; option (my_option) = "my_value"; option (my_option2) = "my_value2"; + + string test = 1; } } """ @@ -199,6 +296,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumValueOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { MY_ENUM_CONSTANT = 0 [(my_option) = "my_value", (my_option2) = "my_value2", (my_option3) = "my_value3"]; } @@ -209,6 +314,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumValueOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { MY_ENUM_CONSTANT = 0 [(my_option3) = "my_value3", (my_option) = "my_value", (my_option2) = "my_value2"]; } @@ -219,6 +332,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.FieldOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { string fieldA = 1 [(my_option) = "my_value", (my_option2) = "my_value2", (my_option3) = "my_value3"]; } @@ -229,6 +350,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.FieldOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { string fieldA = 1 [(my_option3) = "my_value3", (my_option) = "my_value", (my_option2) = "my_value2"]; } @@ -239,11 +368,21 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { enum MyEnum { option (my_option) = "my_value"; option (my_option2) = "my_value2"; option (my_option3) = "my_value3"; + + ACTIVE = 0; } } """ @@ -253,11 +392,21 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { enum MyEnum { option (my_option3) = "my_value3"; option (my_option) = "my_value"; option (my_option2) = "my_value2"; + + ACTIVE = 0; } } """ @@ -267,6 +416,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.FieldOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { message Bar { string fieldA = 1 [(my_option) = "my_value", (my_option2) = "my_value2", (my_option3) = "my_value3"]; @@ -279,6 +436,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.FieldOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + message Foo { message Bar { string fieldA = 1 [(my_option3) = "my_value3", (my_option) = "my_value", (my_option2) = "my_value2"]; @@ -292,6 +457,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.FieldOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { MY_ENUM_CONSTANT = 0; } @@ -306,6 +479,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.FieldOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { MY_ENUM_CONSTANT = 0; } @@ -320,6 +501,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumValueOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { MY_ENUM_CONSTANT = 0 [(my_option) = "my_value", (my_option2) = "my_value2", (my_option3) = "my_value3"]; } @@ -334,6 +523,14 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumValueOptions { + string my_option = 50002; + string my_option2 = 50003; + string my_option3 = 50004; +} + enum MyEnum { MY_ENUM_CONSTANT = 0 [(my_option3) = "my_value3", (my_option) = "my_value", (my_option2) = "my_value2"]; } @@ -348,6 +545,50 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumValueOptions { + string my_option_EnumValue = 50002; + string my_option2_EnumValue = 50003; + string my_option3_EnumValue = 50004; +} + +extend google.protobuf.EnumOptions { + string my_option_Enum = 50002; + string my_option2_Enum = 50003; + string my_option3_Enum = 50004; +} + +extend google.protobuf.FieldOptions { + string my_option_Field = 50002; + string my_option2_Field = 50003; + string my_option3_Field = 50004; +} + +extend google.protobuf.OneofOptions { + string my_option_Oneof = 50002; + string my_option2_Oneof = 50003; + string my_option3_Oneof = 50004; +} + +extend google.protobuf.MessageOptions { + string my_option_Message = 50002; + string my_option2_Message = 50003; + string my_option3_Message = 50004; +} + +extend google.protobuf.MethodOptions { + string my_option_Method = 50002; + string my_option2_Method = 50003; + string my_option3_Method = 50004; +} + +extend google.protobuf.ServiceOptions { + string my_option_Service = 50002; + string my_option2_Service = 50003; + string my_option3_Service = 50004; +} + option cc_generic_services = true; option java_generate_equals_and_hash = true; option java_generic_services = true; @@ -368,47 +609,45 @@ message NestedFoo { string fieldA = 1; - option (my_option) = "my_value"; - option (my_option2) = "my_value2"; + option (my_option_Message) = "my_value"; + option (my_option2_Message) = "my_value2"; } - option (my_option3) = "my_value3"; - option (my_option2) = "my_value2"; - option (my_option) = "my_value"; + option (my_option3_Message) = "my_value3"; + option (my_option2_Message) = "my_value2"; + option (my_option_Message) = "my_value"; oneof my_oneof { - option (my_option3) = "my_value3"; - option (my_option2) = "my_value2"; - option (my_option) = "my_value"; + option (my_option3_Oneof) = "my_value3"; + option (my_option2_Oneof) = "my_value2"; + option (my_option_Oneof) = "my_value"; + + string test = 5; } enum MyEnum { - option (my_option3) = "my_value3"; - option (my_option2) = "my_value2"; - option (my_option) = "my_value"; + option (my_option3_Enum) = "my_value3"; + option (my_option2_Enum) = "my_value2"; + option (my_option_Enum) = "my_value"; + + ACTIVE = 0; } } -extend Foo { - option (my_option3) = "my_value3"; - option (my_option2) = "my_value2"; - option (my_option) = "my_value"; -} - service MyService { - option (my_option3) = "my_value3"; - option (my_option2) = "my_value2"; - option (my_option) = "my_value"; + option (my_option3_Service) = "my_value3"; + option (my_option2_Service) = "my_value2"; + option (my_option_Service) = "my_value"; rpc MyRpc (Foo) returns (Foo) { - option (my_option) = "my_value"; - option (my_option2) = "my_value2"; - option (my_option3) = "my_value3"; + option (my_option_Method) = "my_value"; + option (my_option2_Method) = "my_value2"; + option (my_option3_Method) = "my_value3"; } } @@ -420,6 +659,50 @@ package pkg; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.EnumValueOptions { + string my_option_EnumValue = 50002; + string my_option2_EnumValue = 50003; + string my_option3_EnumValue = 50004; +} + +extend google.protobuf.EnumOptions { + string my_option_Enum = 50002; + string my_option2_Enum = 50003; + string my_option3_Enum = 50004; +} + +extend google.protobuf.FieldOptions { + string my_option_Field = 50002; + string my_option2_Field = 50003; + string my_option3_Field = 50004; +} + +extend google.protobuf.OneofOptions { + string my_option_Oneof = 50002; + string my_option2_Oneof = 50003; + string my_option3_Oneof = 50004; +} + +extend google.protobuf.MessageOptions { + string my_option_Message = 50002; + string my_option2_Message = 50003; + string my_option3_Message = 50004; +} + +extend google.protobuf.MethodOptions { + string my_option_Method = 50002; + string my_option2_Method = 50003; + string my_option3_Method = 50004; +} + +extend google.protobuf.ServiceOptions { + string my_option_Service = 50002; + string my_option2_Service = 50003; + string my_option3_Service = 50004; +} + option cc_generic_services = true; option java_outer_classname = "FooProto"; option optimize_for = SPEED; @@ -440,45 +723,42 @@ message NestedFoo { string fieldA = 1; - option (my_option2) = "my_value2"; - option (my_option) = "my_value"; + option (my_option2_Message) = "my_value2"; + option (my_option_Message) = "my_value"; } - option (my_option2) = "my_value2"; - option (my_option3) = "my_value3"; - option (my_option) = "my_value"; + option (my_option2_Message) = "my_value2"; + option (my_option3_Message) = "my_value3"; + option (my_option_Message) = "my_value"; oneof my_oneof { - option (my_option) = "my_value"; - option (my_option3) = "my_value3"; - option (my_option2) = "my_value2"; + option (my_option_Oneof) = "my_value"; + option (my_option3_Oneof) = "my_value3"; + option (my_option2_Oneof) = "my_value2"; + + string test = 5; } enum MyEnum { - option (my_option) = "my_value"; - option (my_option3) = "my_value3"; - option (my_option2) = "my_value2"; - } -} - + option (my_option_Enum) = "my_value"; + option (my_option3_Enum) = "my_value3"; + option (my_option2_Enum) = "my_value2"; -extend Foo { - option (my_option2) = "my_value2"; - option (my_option3) = "my_value3"; - option (my_option) = "my_value"; + ACTIVE = 0; + } } service MyService { - option (my_option2) = "my_value2"; - option (my_option) = "my_value"; - option (my_option3) = "my_value3"; + option (my_option2_Service) = "my_value2"; + option (my_option_Service) = "my_value"; + option (my_option3_Service) = "my_value3"; rpc MyRpc (Foo) returns (Foo) { - option (my_option2) = "my_value2"; - option (my_option3) = "my_value3"; - option (my_option) = "my_value"; + option (my_option2_Method) = "my_value2"; + option (my_option3_Method) = "my_value3"; + option (my_option_Method) = "my_value"; } } @@ -518,3 +798,39 @@ def test_differently_ordered_options_normalizes_equally(ordered_schema: str, uno normalize(ordered_proto).compare(normalize(unordered_proto), result) assert result.is_compatible() assert normalize(ordered_proto).to_schema() == normalize(unordered_proto).to_schema() + + +@pytest.mark.parametrize( + ("ordered_schema", "unordered_schema"), + ( + (PROTO_WITH_OPTIONS_ORDERED, PROTO_WITH_OPTIONS_UNORDERED), + (PROTO_WITH_OPTIONS_IN_ENUM_ORDERED, PROTO_WITH_OPTIONS_IN_ENUM_UNORDERED), + (PROTO_WITH_OPTIONS_IN_SERVICE_ORDERED, PROTO_WITH_OPTIONS_IN_SERVICE_UNORDERED), + (PROTO_WITH_OPTIONS_IN_RPC_ORDERED, PROTO_WITH_OPTIONS_IN_RPC_UNORDERED), + # Note: This does not work. Is it valid proto? + # (PROTO_WITH_OPTIONS_IN_EXTEND_ORDERED, PROTO_WITH_OPTIONS_IN_EXTEND_UNORDERED), + (PROTO_WITH_OPTIONS_IN_ONEOF_ORDERED, PROTO_WITH_OPTIONS_IN_ONEOF_UNORDERED), + (PROTO_WITH_OPTIONS_IN_ENUM_CONSTANTS_ORDERED, PROTO_WITH_OPTIONS_IN_ENUM_CONSTANTS_UNORDERED), + (PROTO_WITH_OPTIONS_IN_FIELD_OF_MESSAGE_ORDERED, PROTO_WITH_OPTIONS_IN_FIELD_OF_MESSAGE_UNORDERED), + (PROTO_WITH_NEASTED_ENUM_IN_MESSAGE_WITH_OPTIONS_ORDERED, PROTO_WITH_NEASTED_ENUM_IN_MESSAGE_WITH_OPTIONS_UNORDERED), + ( + PROTO_WITH_OPTIONS_IN_FIELD_OF_MESSAGE_WITH_OPTIONS_ORDERED, + PROTO_WITH_OPTIONS_IN_FIELD_OF_MESSAGE_WITH_OPTIONS_UNORDERED, + ), + (PROTO_WITH_OPTIONS_IN_FIELD_OF_ENUM_ORDERED, PROTO_WITH_OPTIONS_IN_FIELD_OF_ENUM_UNORDERED), + ( + PROTO_WITH_OPTIONS_IN_FIELD_OF_ENUM_WITH_OPTIONS_ORDERED, + PROTO_WITH_OPTIONS_IN_FIELD_OF_ENUM_WITH_OPTIONS_UNORDERED, + ), + (PROTO_WITH_COMPLEX_SCHEMA_ORDERED, PROTO_WITH_COMPLEX_SCHEMA_UNORDERED), + ), +) +def test_differently_ordered_options_normalizes_equally_with_protopace(ordered_schema: str, unordered_schema: str) -> None: + ordered_proto = Proto("test.proto", ordered_schema) + unordered_proto = Proto("test.proto", unordered_schema) + + ordered_str = format_proto(ordered_proto) + unordered_str = format_proto(unordered_proto) + + assert ordered_str == unordered_str + check_compatibility(ordered_proto, unordered_proto)