From 05cd9280627149f20881f176f1b629462ebd88fa Mon Sep 17 00:00:00 2001 From: r-vasquez Date: Fri, 6 Dec 2024 12:24:01 -0800 Subject: [PATCH] rpk: add well-known protobuf types to rpk This is the same set of well-known types that both Redpanda and Redpanda Console already support This allows users to use the whole suite and encode/decode using these well-known types. One rpk unit tests verify that the embedded types are the same as the ones in the Redpanda repo, hence the need to trigger rpk tests on changes in pandaproxy/schema_registry/protobuf --- .github/workflows/rpk-build.yml | 2 + src/go/rpk/pkg/serde/BUILD | 1 + src/go/rpk/pkg/serde/embed/BUILD | 38 +++ src/go/rpk/pkg/serde/embed/embed.go | 62 +++++ src/go/rpk/pkg/serde/embed/embed_test.go | 42 +++ .../pkg/serde/embed/protobuf/.clang-format | 2 + .../serde/embed/protobuf/confluent/meta.proto | 27 ++ .../protobuf/confluent/types/decimal.proto | 17 ++ .../embed/protobuf/google/type/README.md | 16 ++ .../google/type/calendar_period.proto | 56 ++++ .../embed/protobuf/google/type/color.proto | 174 ++++++++++++ .../embed/protobuf/google/type/date.proto | 52 ++++ .../embed/protobuf/google/type/datetime.proto | 104 +++++++ .../protobuf/google/type/dayofweek.proto | 50 ++++ .../embed/protobuf/google/type/decimal.proto | 95 +++++++ .../embed/protobuf/google/type/expr.proto | 73 +++++ .../embed/protobuf/google/type/fraction.proto | 33 +++ .../embed/protobuf/google/type/interval.proto | 46 +++ .../embed/protobuf/google/type/latlng.proto | 37 +++ .../protobuf/google/type/localized_text.proto | 36 +++ .../embed/protobuf/google/type/money.proto | 42 +++ .../embed/protobuf/google/type/month.proto | 65 +++++ .../protobuf/google/type/phone_number.proto | 113 ++++++++ .../protobuf/google/type/postal_address.proto | 134 +++++++++ .../protobuf/google/type/quaternion.proto | 94 +++++++ .../protobuf/google/type/timeofday.proto | 44 +++ src/go/rpk/pkg/serde/proto.go | 25 +- src/go/rpk/pkg/serde/proto_test.go | 262 ++++++++++++++++++ src/v/pandaproxy/BUILD | 1 + tests/rptest/clients/rpk.py | 6 +- tests/rptest/tests/rpk_registry_test.py | 75 ++++- 31 files changed, 1813 insertions(+), 11 deletions(-) create mode 100644 src/go/rpk/pkg/serde/embed/BUILD create mode 100644 src/go/rpk/pkg/serde/embed/embed.go create mode 100644 src/go/rpk/pkg/serde/embed/embed_test.go create mode 100644 src/go/rpk/pkg/serde/embed/protobuf/.clang-format create mode 100644 src/go/rpk/pkg/serde/embed/protobuf/confluent/meta.proto create mode 100644 src/go/rpk/pkg/serde/embed/protobuf/confluent/types/decimal.proto create mode 100644 src/go/rpk/pkg/serde/embed/protobuf/google/type/README.md create mode 100644 src/go/rpk/pkg/serde/embed/protobuf/google/type/calendar_period.proto create mode 100644 src/go/rpk/pkg/serde/embed/protobuf/google/type/color.proto create mode 100644 src/go/rpk/pkg/serde/embed/protobuf/google/type/date.proto create mode 100644 src/go/rpk/pkg/serde/embed/protobuf/google/type/datetime.proto create mode 100644 src/go/rpk/pkg/serde/embed/protobuf/google/type/dayofweek.proto create mode 100644 src/go/rpk/pkg/serde/embed/protobuf/google/type/decimal.proto create mode 100644 src/go/rpk/pkg/serde/embed/protobuf/google/type/expr.proto create mode 100644 src/go/rpk/pkg/serde/embed/protobuf/google/type/fraction.proto create mode 100644 src/go/rpk/pkg/serde/embed/protobuf/google/type/interval.proto create mode 100644 src/go/rpk/pkg/serde/embed/protobuf/google/type/latlng.proto create mode 100644 src/go/rpk/pkg/serde/embed/protobuf/google/type/localized_text.proto create mode 100644 src/go/rpk/pkg/serde/embed/protobuf/google/type/money.proto create mode 100644 src/go/rpk/pkg/serde/embed/protobuf/google/type/month.proto create mode 100644 src/go/rpk/pkg/serde/embed/protobuf/google/type/phone_number.proto create mode 100644 src/go/rpk/pkg/serde/embed/protobuf/google/type/postal_address.proto create mode 100644 src/go/rpk/pkg/serde/embed/protobuf/google/type/quaternion.proto create mode 100644 src/go/rpk/pkg/serde/embed/protobuf/google/type/timeofday.proto diff --git a/.github/workflows/rpk-build.yml b/.github/workflows/rpk-build.yml index b762a90a4991..1b42e4113b09 100644 --- a/.github/workflows/rpk-build.yml +++ b/.github/workflows/rpk-build.yml @@ -13,10 +13,12 @@ on: branches: [dev] paths: - 'src/go/rpk/**' + - 'src/v/pandaproxy/schema_registry/protobuf/**' - '.github/workflows/rpk-build.yml' pull_request: paths: - 'src/go/rpk/**' + - 'src/v/pandaproxy/schema_registry/protobuf/**' - '.github/workflows/rpk-build.yml' jobs: test: diff --git a/src/go/rpk/pkg/serde/BUILD b/src/go/rpk/pkg/serde/BUILD index 240f96c40e00..a7fb6b167a98 100644 --- a/src/go/rpk/pkg/serde/BUILD +++ b/src/go/rpk/pkg/serde/BUILD @@ -11,6 +11,7 @@ go_library( importpath = "github.com/redpanda-data/redpanda/src/go/rpk/pkg/serde", visibility = ["//visibility:public"], deps = [ + "//src/go/rpk/pkg/serde/embed", "@com_github_bufbuild_protocompile//:protocompile", "@com_github_bufbuild_protocompile//linker", "@com_github_hamba_avro_v2//:avro", diff --git a/src/go/rpk/pkg/serde/embed/BUILD b/src/go/rpk/pkg/serde/embed/BUILD new file mode 100644 index 000000000000..303033b8187c --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/BUILD @@ -0,0 +1,38 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "embed", + srcs = ["embed.go"], + embedsrcs = [ + "protobuf/confluent/meta.proto", + "protobuf/confluent/types/decimal.proto", + "protobuf/google/type/calendar_period.proto", + "protobuf/google/type/color.proto", + "protobuf/google/type/date.proto", + "protobuf/google/type/datetime.proto", + "protobuf/google/type/dayofweek.proto", + "protobuf/google/type/decimal.proto", + "protobuf/google/type/expr.proto", + "protobuf/google/type/fraction.proto", + "protobuf/google/type/interval.proto", + "protobuf/google/type/latlng.proto", + "protobuf/google/type/localized_text.proto", + "protobuf/google/type/money.proto", + "protobuf/google/type/month.proto", + "protobuf/google/type/phone_number.proto", + "protobuf/google/type/postal_address.proto", + "protobuf/google/type/quaternion.proto", + "protobuf/google/type/timeofday.proto", + ], + importpath = "github.com/redpanda-data/redpanda/src/go/rpk/pkg/serde/embed", + visibility = ["//visibility:public"], +) + +go_test( + name = "embed_test", + size = "small", + srcs = ["embed_test.go"], + data = ["//src/v/pandaproxy:schema_registry_protos"], + embed = [":embed"], + deps = ["@com_github_stretchr_testify//require"], +) diff --git a/src/go/rpk/pkg/serde/embed/embed.go b/src/go/rpk/pkg/serde/embed/embed.go new file mode 100644 index 000000000000..07d7bd1ba502 --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/embed.go @@ -0,0 +1,62 @@ +// Copyright 2024 Redpanda Data, Inc. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.md +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0 + +package embed + +import ( + "embed" + "io/fs" + "path/filepath" + "sync" +) + +//go:embed protobuf/google/type/*.proto protobuf/confluent/*.proto protobuf/confluent/types/*.proto +var content embed.FS + +var ( + once sync.Once + protoMap map[string]string +) + +// CommonProtoFiles returns the file system representation of the common +// protobuf types. +func CommonProtoFiles() (fs.FS, error) { + return fs.Sub(content, "protobuf") +} + +// CommonProtoFileMap returns the map representation of the common protobuf +// types. This is useful for protoreflect parsing. +func CommonProtoFileMap() (map[string]string, error) { + protoFS, err := CommonProtoFiles() + if err != nil { + return nil, err + } + + once.Do(func() { + protoMap = make(map[string]string) + err = fs.WalkDir(protoFS, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || filepath.Ext(path) != ".proto" { + return nil + } + data, err := fs.ReadFile(protoFS, path) + if err == nil { + protoMap[path] = string(data) + } + return nil + }) + }) + + if err != nil { + return nil, err + } + return protoMap, err +} diff --git a/src/go/rpk/pkg/serde/embed/embed_test.go b/src/go/rpk/pkg/serde/embed/embed_test.go new file mode 100644 index 000000000000..e96c728f3d0b --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/embed_test.go @@ -0,0 +1,42 @@ +package embed + +import ( + "io/fs" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestEmbeddedFiles(t *testing.T) { + t.Run("Test Embedded files in rpk, equal to Redpanda", func(t *testing.T) { + // /src/v/pandaproxy/schema_registry/protobuf + redpandaProtoFS := os.DirFS("../../../../../v/pandaproxy/schema_registry/protobuf/") + redpandaMap := make(map[string]string) + err := fs.WalkDir(redpandaProtoFS, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || filepath.Ext(path) != ".proto" { + return nil + } + data, err := fs.ReadFile(redpandaProtoFS, path) + if err == nil { + redpandaMap[path] = string(data) + } + return nil + }) + + embeddedMap, err := CommonProtoFileMap() + require.NoError(t, err) + + for path, embedContent := range embeddedMap { + if rpContent, ok := redpandaMap[path]; ok { + require.Equalf(t, rpContent, embedContent, "Contents of %v have changed vs the embedded rpk files", path) + } else { + t.Fatalf("%s not found in Redpanda files", path) + } + } + }) +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/.clang-format b/src/go/rpk/pkg/serde/embed/protobuf/.clang-format new file mode 100644 index 000000000000..ef9566ac8a7a --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/.clang-format @@ -0,0 +1,2 @@ +Language: Proto +DisableFormat: true diff --git a/src/go/rpk/pkg/serde/embed/protobuf/confluent/meta.proto b/src/go/rpk/pkg/serde/embed/protobuf/confluent/meta.proto new file mode 100644 index 000000000000..4439139d0c68 --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/confluent/meta.proto @@ -0,0 +1,27 @@ +syntax = "proto3"; +package confluent; + +import "google/protobuf/descriptor.proto"; + +option go_package="../confluent"; + +message Meta { + string doc = 1; + map params = 2; +} + +extend google.protobuf.FileOptions { + Meta file_meta = 1088; +} +extend google.protobuf.MessageOptions { + Meta message_meta = 1088; +} +extend google.protobuf.FieldOptions { + Meta field_meta = 1088; +} +extend google.protobuf.EnumOptions { + Meta enum_meta = 1088; +} +extend google.protobuf.EnumValueOptions { + Meta enum_value_meta = 1088; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/confluent/types/decimal.proto b/src/go/rpk/pkg/serde/embed/protobuf/confluent/types/decimal.proto new file mode 100644 index 000000000000..75d8b9b46f0a --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/confluent/types/decimal.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +package confluent.type; + +option go_package="../types"; + +message Decimal { + + // The two's-complement representation of the unscaled integer value in big-endian byte order + bytes value = 1; + + // The precision + uint32 precision = 2; + + // The scale + int32 scale = 3; +} \ No newline at end of file diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/README.md b/src/go/rpk/pkg/serde/embed/protobuf/google/type/README.md new file mode 100644 index 000000000000..6caf02cf1f5e --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/README.md @@ -0,0 +1,16 @@ +# Google Common Types + +This package contains definitions of common types for Google APIs. +All types defined in this package are suitable for different APIs to +exchange data, and will never break binary compatibility. They should +have design quality comparable to major programming languages like +Java and C#. + +NOTE: Some common types are defined in the package `google.protobuf` +as they are directly supported by Protocol Buffers compiler and +runtime. Those types are called Well-Known Types. + +## Java Utilities + +A set of Java utilities for the Common Types are provided in the +`//java/com/google/type/util/` package. \ No newline at end of file diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/calendar_period.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/calendar_period.proto new file mode 100644 index 000000000000..82f5690b752b --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/calendar_period.proto @@ -0,0 +1,56 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/calendarperiod;calendarperiod"; +option java_multiple_files = true; +option java_outer_classname = "CalendarPeriodProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// A `CalendarPeriod` represents the abstract concept of a time period that has +// a canonical start. Grammatically, "the start of the current +// `CalendarPeriod`." All calendar times begin at midnight UTC. +enum CalendarPeriod { + // Undefined period, raises an error. + CALENDAR_PERIOD_UNSPECIFIED = 0; + + // A day. + DAY = 1; + + // A week. Weeks begin on Monday, following + // [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date). + WEEK = 2; + + // A fortnight. The first calendar fortnight of the year begins at the start + // of week 1 according to + // [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date). + FORTNIGHT = 3; + + // A month. + MONTH = 4; + + // A quarter. Quarters start on dates 1-Jan, 1-Apr, 1-Jul, and 1-Oct of each + // year. + QUARTER = 5; + + // A half-year. Half-years start on dates 1-Jan and 1-Jul. + HALF = 6; + + // A year. + YEAR = 7; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/color.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/color.proto new file mode 100644 index 000000000000..5dc85a6a3856 --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/color.proto @@ -0,0 +1,174 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +import "google/protobuf/wrappers.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/color;color"; +option java_multiple_files = true; +option java_outer_classname = "ColorProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a color in the RGBA color space. This representation is designed +// for simplicity of conversion to/from color representations in various +// languages over compactness. For example, the fields of this representation +// can be trivially provided to the constructor of `java.awt.Color` in Java; it +// can also be trivially provided to UIColor's `+colorWithRed:green:blue:alpha` +// method in iOS; and, with just a little work, it can be easily formatted into +// a CSS `rgba()` string in JavaScript. +// +// This reference page doesn't carry information about the absolute color +// space +// that should be used to interpret the RGB value (e.g. sRGB, Adobe RGB, +// DCI-P3, BT.2020, etc.). By default, applications should assume the sRGB color +// space. +// +// When color equality needs to be decided, implementations, unless +// documented otherwise, treat two colors as equal if all their red, +// green, blue, and alpha values each differ by at most 1e-5. +// +// Example (Java): +// +// import com.google.type.Color; +// +// // ... +// public static java.awt.Color fromProto(Color protocolor) { +// float alpha = protocolor.hasAlpha() +// ? protocolor.getAlpha().getValue() +// : 1.0; +// +// return new java.awt.Color( +// protocolor.getRed(), +// protocolor.getGreen(), +// protocolor.getBlue(), +// alpha); +// } +// +// public static Color toProto(java.awt.Color color) { +// float red = (float) color.getRed(); +// float green = (float) color.getGreen(); +// float blue = (float) color.getBlue(); +// float denominator = 255.0; +// Color.Builder resultBuilder = +// Color +// .newBuilder() +// .setRed(red / denominator) +// .setGreen(green / denominator) +// .setBlue(blue / denominator); +// int alpha = color.getAlpha(); +// if (alpha != 255) { +// result.setAlpha( +// FloatValue +// .newBuilder() +// .setValue(((float) alpha) / denominator) +// .build()); +// } +// return resultBuilder.build(); +// } +// // ... +// +// Example (iOS / Obj-C): +// +// // ... +// static UIColor* fromProto(Color* protocolor) { +// float red = [protocolor red]; +// float green = [protocolor green]; +// float blue = [protocolor blue]; +// FloatValue* alpha_wrapper = [protocolor alpha]; +// float alpha = 1.0; +// if (alpha_wrapper != nil) { +// alpha = [alpha_wrapper value]; +// } +// return [UIColor colorWithRed:red green:green blue:blue alpha:alpha]; +// } +// +// static Color* toProto(UIColor* color) { +// CGFloat red, green, blue, alpha; +// if (![color getRed:&red green:&green blue:&blue alpha:&alpha]) { +// return nil; +// } +// Color* result = [[Color alloc] init]; +// [result setRed:red]; +// [result setGreen:green]; +// [result setBlue:blue]; +// if (alpha <= 0.9999) { +// [result setAlpha:floatWrapperWithValue(alpha)]; +// } +// [result autorelease]; +// return result; +// } +// // ... +// +// Example (JavaScript): +// +// // ... +// +// var protoToCssColor = function(rgb_color) { +// var redFrac = rgb_color.red || 0.0; +// var greenFrac = rgb_color.green || 0.0; +// var blueFrac = rgb_color.blue || 0.0; +// var red = Math.floor(redFrac * 255); +// var green = Math.floor(greenFrac * 255); +// var blue = Math.floor(blueFrac * 255); +// +// if (!('alpha' in rgb_color)) { +// return rgbToCssColor(red, green, blue); +// } +// +// var alphaFrac = rgb_color.alpha.value || 0.0; +// var rgbParams = [red, green, blue].join(','); +// return ['rgba(', rgbParams, ',', alphaFrac, ')'].join(''); +// }; +// +// var rgbToCssColor = function(red, green, blue) { +// var rgbNumber = new Number((red << 16) | (green << 8) | blue); +// var hexString = rgbNumber.toString(16); +// var missingZeros = 6 - hexString.length; +// var resultBuilder = ['#']; +// for (var i = 0; i < missingZeros; i++) { +// resultBuilder.push('0'); +// } +// resultBuilder.push(hexString); +// return resultBuilder.join(''); +// }; +// +// // ... +message Color { + // The amount of red in the color as a value in the interval [0, 1]. + float red = 1; + + // The amount of green in the color as a value in the interval [0, 1]. + float green = 2; + + // The amount of blue in the color as a value in the interval [0, 1]. + float blue = 3; + + // The fraction of this color that should be applied to the pixel. That is, + // the final pixel color is defined by the equation: + // + // `pixel color = alpha * (this color) + (1.0 - alpha) * (background color)` + // + // This means that a value of 1.0 corresponds to a solid color, whereas + // a value of 0.0 corresponds to a completely transparent color. This + // uses a wrapper message rather than a simple float scalar so that it is + // possible to distinguish between a default value and the value being unset. + // If omitted, this color object is rendered as a solid color + // (as if the alpha value had been explicitly given a value of 1.0). + google.protobuf.FloatValue alpha = 4; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/date.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/date.proto new file mode 100644 index 000000000000..e4e730e6f5a9 --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/date.proto @@ -0,0 +1,52 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/date;date"; +option java_multiple_files = true; +option java_outer_classname = "DateProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a whole or partial calendar date, such as a birthday. The time of +// day and time zone are either specified elsewhere or are insignificant. The +// date is relative to the Gregorian Calendar. This can represent one of the +// following: +// +// * A full date, with non-zero year, month, and day values +// * A month and day value, with a zero year, such as an anniversary +// * A year on its own, with zero month and day values +// * A year and month value, with a zero day, such as a credit card expiration +// date +// +// Related types are [google.type.TimeOfDay][google.type.TimeOfDay] and +// `google.protobuf.Timestamp`. +message Date { + // Year of the date. Must be from 1 to 9999, or 0 to specify a date without + // a year. + int32 year = 1; + + // Month of a year. Must be from 1 to 12, or 0 to specify a year without a + // month and day. + int32 month = 2; + + // Day of a month. Must be from 1 to 31 and valid for the year and month, or 0 + // to specify a year by itself or a year and month where the day isn't + // significant. + int32 day = 3; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/datetime.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/datetime.proto new file mode 100644 index 000000000000..cfed85d70a10 --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/datetime.proto @@ -0,0 +1,104 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +import "google/protobuf/duration.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/datetime;datetime"; +option java_multiple_files = true; +option java_outer_classname = "DateTimeProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents civil time (or occasionally physical time). +// +// This type can represent a civil time in one of a few possible ways: +// +// * When utc_offset is set and time_zone is unset: a civil time on a calendar +// day with a particular offset from UTC. +// * When time_zone is set and utc_offset is unset: a civil time on a calendar +// day in a particular time zone. +// * When neither time_zone nor utc_offset is set: a civil time on a calendar +// day in local time. +// +// The date is relative to the Proleptic Gregorian Calendar. +// +// If year is 0, the DateTime is considered not to have a specific year. month +// and day must have valid, non-zero values. +// +// This type may also be used to represent a physical time if all the date and +// time fields are set and either case of the `time_offset` oneof is set. +// Consider using `Timestamp` message for physical time instead. If your use +// case also would like to store the user's timezone, that can be done in +// another field. +// +// This type is more flexible than some applications may want. Make sure to +// document and validate your application's limitations. +message DateTime { + // Optional. Year of date. Must be from 1 to 9999, or 0 if specifying a + // datetime without a year. + int32 year = 1; + + // Required. Month of year. Must be from 1 to 12. + int32 month = 2; + + // Required. Day of month. Must be from 1 to 31 and valid for the year and + // month. + int32 day = 3; + + // Required. Hours of day in 24 hour format. Should be from 0 to 23. An API + // may choose to allow the value "24:00:00" for scenarios like business + // closing time. + int32 hours = 4; + + // Required. Minutes of hour of day. Must be from 0 to 59. + int32 minutes = 5; + + // Required. Seconds of minutes of the time. Must normally be from 0 to 59. An + // API may allow the value 60 if it allows leap-seconds. + int32 seconds = 6; + + // Required. Fractions of seconds in nanoseconds. Must be from 0 to + // 999,999,999. + int32 nanos = 7; + + // Optional. Specifies either the UTC offset or the time zone of the DateTime. + // Choose carefully between them, considering that time zone data may change + // in the future (for example, a country modifies their DST start/end dates, + // and future DateTimes in the affected range had already been stored). + // If omitted, the DateTime is considered to be in local time. + oneof time_offset { + // UTC offset. Must be whole seconds, between -18 hours and +18 hours. + // For example, a UTC offset of -4:00 would be represented as + // { seconds: -14400 }. + google.protobuf.Duration utc_offset = 8; + + // Time zone. + TimeZone time_zone = 9; + } +} + +// Represents a time zone from the +// [IANA Time Zone Database](https://www.iana.org/time-zones). +message TimeZone { + // IANA Time Zone Database time zone, e.g. "America/New_York". + string id = 1; + + // Optional. IANA Time Zone Database version number, e.g. "2019a". + string version = 2; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/dayofweek.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/dayofweek.proto new file mode 100644 index 000000000000..4c80c62ec0b4 --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/dayofweek.proto @@ -0,0 +1,50 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/dayofweek;dayofweek"; +option java_multiple_files = true; +option java_outer_classname = "DayOfWeekProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a day of the week. +enum DayOfWeek { + // The day of the week is unspecified. + DAY_OF_WEEK_UNSPECIFIED = 0; + + // Monday + MONDAY = 1; + + // Tuesday + TUESDAY = 2; + + // Wednesday + WEDNESDAY = 3; + + // Thursday + THURSDAY = 4; + + // Friday + FRIDAY = 5; + + // Saturday + SATURDAY = 6; + + // Sunday + SUNDAY = 7; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/decimal.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/decimal.proto new file mode 100644 index 000000000000..beb18a5d8dd2 --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/decimal.proto @@ -0,0 +1,95 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/decimal;decimal"; +option java_multiple_files = true; +option java_outer_classname = "DecimalProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// A representation of a decimal value, such as 2.5. Clients may convert values +// into language-native decimal formats, such as Java's [BigDecimal][] or +// Python's [decimal.Decimal][]. +// +// [BigDecimal]: +// https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/math/BigDecimal.html +// [decimal.Decimal]: https://docs.python.org/3/library/decimal.html +message Decimal { + // The decimal value, as a string. + // + // The string representation consists of an optional sign, `+` (`U+002B`) + // or `-` (`U+002D`), followed by a sequence of zero or more decimal digits + // ("the integer"), optionally followed by a fraction, optionally followed + // by an exponent. + // + // The fraction consists of a decimal point followed by zero or more decimal + // digits. The string must contain at least one digit in either the integer + // or the fraction. The number formed by the sign, the integer and the + // fraction is referred to as the significand. + // + // The exponent consists of the character `e` (`U+0065`) or `E` (`U+0045`) + // followed by one or more decimal digits. + // + // Services **should** normalize decimal values before storing them by: + // + // - Removing an explicitly-provided `+` sign (`+2.5` -> `2.5`). + // - Replacing a zero-length integer value with `0` (`.5` -> `0.5`). + // - Coercing the exponent character to lower-case (`2.5E8` -> `2.5e8`). + // - Removing an explicitly-provided zero exponent (`2.5e0` -> `2.5`). + // + // Services **may** perform additional normalization based on its own needs + // and the internal decimal implementation selected, such as shifting the + // decimal point and exponent value together (example: `2.5e-1` <-> `0.25`). + // Additionally, services **may** preserve trailing zeroes in the fraction + // to indicate increased precision, but are not required to do so. + // + // Note that only the `.` character is supported to divide the integer + // and the fraction; `,` **should not** be supported regardless of locale. + // Additionally, thousand separators **should not** be supported. If a + // service does support them, values **must** be normalized. + // + // The ENBF grammar is: + // + // DecimalString = + // [Sign] Significand [Exponent]; + // + // Sign = '+' | '-'; + // + // Significand = + // Digits ['.'] [Digits] | [Digits] '.' Digits; + // + // Exponent = ('e' | 'E') [Sign] Digits; + // + // Digits = { '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' }; + // + // Services **should** clearly document the range of supported values, the + // maximum supported precision (total number of digits), and, if applicable, + // the scale (number of digits after the decimal point), as well as how it + // behaves when receiving out-of-bounds values. + // + // Services **may** choose to accept values passed as input even when the + // value has a higher precision or scale than the service supports, and + // **should** round the value to fit the supported scale. Alternatively, the + // service **may** error with `400 Bad Request` (`INVALID_ARGUMENT` in gRPC) + // if precision would be lost. + // + // Services **should** error with `400 Bad Request` (`INVALID_ARGUMENT` in + // gRPC) if the service receives a value outside of the supported range. + string value = 1; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/expr.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/expr.proto new file mode 100644 index 000000000000..af0778cf958c --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/expr.proto @@ -0,0 +1,73 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/expr;expr"; +option java_multiple_files = true; +option java_outer_classname = "ExprProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a textual expression in the Common Expression Language (CEL) +// syntax. CEL is a C-like expression language. The syntax and semantics of CEL +// are documented at https://github.com/google/cel-spec. +// +// Example (Comparison): +// +// title: "Summary size limit" +// description: "Determines if a summary is less than 100 chars" +// expression: "document.summary.size() < 100" +// +// Example (Equality): +// +// title: "Requestor is owner" +// description: "Determines if requestor is the document owner" +// expression: "document.owner == request.auth.claims.email" +// +// Example (Logic): +// +// title: "Public documents" +// description: "Determine whether the document should be publicly visible" +// expression: "document.type != 'private' && document.type != 'internal'" +// +// Example (Data Manipulation): +// +// title: "Notification string" +// description: "Create a notification string with a timestamp." +// expression: "'New message received at ' + string(document.create_time)" +// +// The exact variables and functions that may be referenced within an expression +// are determined by the service that evaluates it. See the service +// documentation for additional information. +message Expr { + // Textual representation of an expression in Common Expression Language + // syntax. + string expression = 1; + + // Optional. Title for the expression, i.e. a short string describing + // its purpose. This can be used e.g. in UIs which allow to enter the + // expression. + string title = 2; + + // Optional. Description of the expression. This is a longer text which + // describes the expression, e.g. when hovered over it in a UI. + string description = 3; + + // Optional. String indicating the location of the expression for error + // reporting, e.g. a file name and a position in the file. + string location = 4; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/fraction.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/fraction.proto new file mode 100644 index 000000000000..6c5ae6e2a25d --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/fraction.proto @@ -0,0 +1,33 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/fraction;fraction"; +option java_multiple_files = true; +option java_outer_classname = "FractionProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a fraction in terms of a numerator divided by a denominator. +message Fraction { + // The numerator in the fraction, e.g. 2 in 2/3. + int64 numerator = 1; + + // The value by which the numerator is divided, e.g. 3 in 2/3. Must be + // positive. + int64 denominator = 2; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/interval.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/interval.proto new file mode 100644 index 000000000000..9702324cd4e8 --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/interval.proto @@ -0,0 +1,46 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +import "google/protobuf/timestamp.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/interval;interval"; +option java_multiple_files = true; +option java_outer_classname = "IntervalProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a time interval, encoded as a Timestamp start (inclusive) and a +// Timestamp end (exclusive). +// +// The start must be less than or equal to the end. +// When the start equals the end, the interval is empty (matches no time). +// When both start and end are unspecified, the interval matches any time. +message Interval { + // Optional. Inclusive start of the interval. + // + // If specified, a Timestamp matching this interval will have to be the same + // or after the start. + google.protobuf.Timestamp start_time = 1; + + // Optional. Exclusive end of the interval. + // + // If specified, a Timestamp matching this interval will have to be before the + // end. + google.protobuf.Timestamp end_time = 2; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/latlng.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/latlng.proto new file mode 100644 index 000000000000..9231456e328f --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/latlng.proto @@ -0,0 +1,37 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/latlng;latlng"; +option java_multiple_files = true; +option java_outer_classname = "LatLngProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// An object that represents a latitude/longitude pair. This is expressed as a +// pair of doubles to represent degrees latitude and degrees longitude. Unless +// specified otherwise, this must conform to the +// WGS84 +// standard. Values must be within normalized ranges. +message LatLng { + // The latitude in degrees. It must be in the range [-90.0, +90.0]. + double latitude = 1; + + // The longitude in degrees. It must be in the range [-180.0, +180.0]. + double longitude = 2; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/localized_text.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/localized_text.proto new file mode 100644 index 000000000000..5c6922b8c0cd --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/localized_text.proto @@ -0,0 +1,36 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/localized_text;localized_text"; +option java_multiple_files = true; +option java_outer_classname = "LocalizedTextProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Localized variant of a text in a particular language. +message LocalizedText { + // Localized string in the language corresponding to `language_code' below. + string text = 1; + + // The text's BCP-47 language code, such as "en-US" or "sr-Latn". + // + // For more information, see + // http://www.unicode.org/reports/tr35/#Unicode_locale_identifier. + string language_code = 2; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/money.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/money.proto new file mode 100644 index 000000000000..98d6494e4210 --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/money.proto @@ -0,0 +1,42 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/money;money"; +option java_multiple_files = true; +option java_outer_classname = "MoneyProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents an amount of money with its currency type. +message Money { + // The three-letter currency code defined in ISO 4217. + string currency_code = 1; + + // The whole units of the amount. + // For example if `currencyCode` is `"USD"`, then 1 unit is one US dollar. + int64 units = 2; + + // Number of nano (10^-9) units of the amount. + // The value must be between -999,999,999 and +999,999,999 inclusive. + // If `units` is positive, `nanos` must be positive or zero. + // If `units` is zero, `nanos` can be positive, zero, or negative. + // If `units` is negative, `nanos` must be negative or zero. + // For example $-1.75 is represented as `units`=-1 and `nanos`=-750,000,000. + int32 nanos = 3; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/month.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/month.proto new file mode 100644 index 000000000000..99e7551b1416 --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/month.proto @@ -0,0 +1,65 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/month;month"; +option java_multiple_files = true; +option java_outer_classname = "MonthProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a month in the Gregorian calendar. +enum Month { + // The unspecified month. + MONTH_UNSPECIFIED = 0; + + // The month of January. + JANUARY = 1; + + // The month of February. + FEBRUARY = 2; + + // The month of March. + MARCH = 3; + + // The month of April. + APRIL = 4; + + // The month of May. + MAY = 5; + + // The month of June. + JUNE = 6; + + // The month of July. + JULY = 7; + + // The month of August. + AUGUST = 8; + + // The month of September. + SEPTEMBER = 9; + + // The month of October. + OCTOBER = 10; + + // The month of November. + NOVEMBER = 11; + + // The month of December. + DECEMBER = 12; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/phone_number.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/phone_number.proto new file mode 100644 index 000000000000..7bbb7d873229 --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/phone_number.proto @@ -0,0 +1,113 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/phone_number;phone_number"; +option java_multiple_files = true; +option java_outer_classname = "PhoneNumberProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// An object representing a phone number, suitable as an API wire format. +// +// This representation: +// +// - should not be used for locale-specific formatting of a phone number, such +// as "+1 (650) 253-0000 ext. 123" +// +// - is not designed for efficient storage +// - may not be suitable for dialing - specialized libraries (see references) +// should be used to parse the number for that purpose +// +// To do something meaningful with this number, such as format it for various +// use-cases, convert it to an `i18n.phonenumbers.PhoneNumber` object first. +// +// For instance, in Java this would be: +// +// com.google.type.PhoneNumber wireProto = +// com.google.type.PhoneNumber.newBuilder().build(); +// com.google.i18n.phonenumbers.Phonenumber.PhoneNumber phoneNumber = +// PhoneNumberUtil.getInstance().parse(wireProto.getE164Number(), "ZZ"); +// if (!wireProto.getExtension().isEmpty()) { +// phoneNumber.setExtension(wireProto.getExtension()); +// } +// +// Reference(s): +// - https://github.com/google/libphonenumber +message PhoneNumber { + // An object representing a short code, which is a phone number that is + // typically much shorter than regular phone numbers and can be used to + // address messages in MMS and SMS systems, as well as for abbreviated dialing + // (e.g. "Text 611 to see how many minutes you have remaining on your plan."). + // + // Short codes are restricted to a region and are not internationally + // dialable, which means the same short code can exist in different regions, + // with different usage and pricing, even if those regions share the same + // country calling code (e.g. US and CA). + message ShortCode { + // Required. The BCP-47 region code of the location where calls to this + // short code can be made, such as "US" and "BB". + // + // Reference(s): + // - http://www.unicode.org/reports/tr35/#unicode_region_subtag + string region_code = 1; + + // Required. The short code digits, without a leading plus ('+') or country + // calling code, e.g. "611". + string number = 2; + } + + // Required. Either a regular number, or a short code. New fields may be + // added to the oneof below in the future, so clients should ignore phone + // numbers for which none of the fields they coded against are set. + oneof kind { + // The phone number, represented as a leading plus sign ('+'), followed by a + // phone number that uses a relaxed ITU E.164 format consisting of the + // country calling code (1 to 3 digits) and the subscriber number, with no + // additional spaces or formatting, e.g.: + // - correct: "+15552220123" + // - incorrect: "+1 (555) 222-01234 x123". + // + // The ITU E.164 format limits the latter to 12 digits, but in practice not + // all countries respect that, so we relax that restriction here. + // National-only numbers are not allowed. + // + // References: + // - https://www.itu.int/rec/T-REC-E.164-201011-I + // - https://en.wikipedia.org/wiki/E.164. + // - https://en.wikipedia.org/wiki/List_of_country_calling_codes + string e164_number = 1; + + // A short code. + // + // Reference(s): + // - https://en.wikipedia.org/wiki/Short_code + ShortCode short_code = 2; + } + + // The phone number's extension. The extension is not standardized in ITU + // recommendations, except for being defined as a series of numbers with a + // maximum length of 40 digits. Other than digits, some other dialing + // characters such as ',' (indicating a wait) or '#' may be stored here. + // + // Note that no regions currently use extensions with short codes, so this + // field is normally only set in conjunction with an E.164 number. It is held + // separately from the E.164 number to allow for short code extensions in the + // future. + string extension = 3; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/postal_address.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/postal_address.proto new file mode 100644 index 000000000000..c57c7c31a2cc --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/postal_address.proto @@ -0,0 +1,134 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/postaladdress;postaladdress"; +option java_multiple_files = true; +option java_outer_classname = "PostalAddressProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a postal address, e.g. for postal delivery or payments addresses. +// Given a postal address, a postal service can deliver items to a premise, P.O. +// Box or similar. +// It is not intended to model geographical locations (roads, towns, +// mountains). +// +// In typical usage an address would be created via user input or from importing +// existing data, depending on the type of process. +// +// Advice on address input / editing: +// - Use an i18n-ready address widget such as +// https://github.com/google/libaddressinput) +// - Users should not be presented with UI elements for input or editing of +// fields outside countries where that field is used. +// +// For more guidance on how to use this schema, please see: +// https://support.google.com/business/answer/6397478 +message PostalAddress { + // The schema revision of the `PostalAddress`. This must be set to 0, which is + // the latest revision. + // + // All new revisions **must** be backward compatible with old revisions. + int32 revision = 1; + + // Required. CLDR region code of the country/region of the address. This + // is never inferred and it is up to the user to ensure the value is + // correct. See http://cldr.unicode.org/ and + // http://www.unicode.org/cldr/charts/30/supplemental/territory_information.html + // for details. Example: "CH" for Switzerland. + string region_code = 2; + + // Optional. BCP-47 language code of the contents of this address (if + // known). This is often the UI language of the input form or is expected + // to match one of the languages used in the address' country/region, or their + // transliterated equivalents. + // This can affect formatting in certain countries, but is not critical + // to the correctness of the data and will never affect any validation or + // other non-formatting related operations. + // + // If this value is not known, it should be omitted (rather than specifying a + // possibly incorrect default). + // + // Examples: "zh-Hant", "ja", "ja-Latn", "en". + string language_code = 3; + + // Optional. Postal code of the address. Not all countries use or require + // postal codes to be present, but where they are used, they may trigger + // additional validation with other parts of the address (e.g. state/zip + // validation in the U.S.A.). + string postal_code = 4; + + // Optional. Additional, country-specific, sorting code. This is not used + // in most regions. Where it is used, the value is either a string like + // "CEDEX", optionally followed by a number (e.g. "CEDEX 7"), or just a number + // alone, representing the "sector code" (Jamaica), "delivery area indicator" + // (Malawi) or "post office indicator" (e.g. Côte d'Ivoire). + string sorting_code = 5; + + // Optional. Highest administrative subdivision which is used for postal + // addresses of a country or region. + // For example, this can be a state, a province, an oblast, or a prefecture. + // Specifically, for Spain this is the province and not the autonomous + // community (e.g. "Barcelona" and not "Catalonia"). + // Many countries don't use an administrative area in postal addresses. E.g. + // in Switzerland this should be left unpopulated. + string administrative_area = 6; + + // Optional. Generally refers to the city/town portion of the address. + // Examples: US city, IT comune, UK post town. + // In regions of the world where localities are not well defined or do not fit + // into this structure well, leave locality empty and use address_lines. + string locality = 7; + + // Optional. Sublocality of the address. + // For example, this can be neighborhoods, boroughs, districts. + string sublocality = 8; + + // Unstructured address lines describing the lower levels of an address. + // + // Because values in address_lines do not have type information and may + // sometimes contain multiple values in a single field (e.g. + // "Austin, TX"), it is important that the line order is clear. The order of + // address lines should be "envelope order" for the country/region of the + // address. In places where this can vary (e.g. Japan), address_language is + // used to make it explicit (e.g. "ja" for large-to-small ordering and + // "ja-Latn" or "en" for small-to-large). This way, the most specific line of + // an address can be selected based on the language. + // + // The minimum permitted structural representation of an address consists + // of a region_code with all remaining information placed in the + // address_lines. It would be possible to format such an address very + // approximately without geocoding, but no semantic reasoning could be + // made about any of the address components until it was at least + // partially resolved. + // + // Creating an address only containing a region_code and address_lines, and + // then geocoding is the recommended way to handle completely unstructured + // addresses (as opposed to guessing which parts of the address should be + // localities or administrative areas). + repeated string address_lines = 9; + + // Optional. The recipient at the address. + // This field may, under certain circumstances, contain multiline information. + // For example, it might contain "care of" information. + repeated string recipients = 10; + + // Optional. The name of the organization at the address. + string organization = 11; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/quaternion.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/quaternion.proto new file mode 100644 index 000000000000..dfb822deff16 --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/quaternion.proto @@ -0,0 +1,94 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/quaternion;quaternion"; +option java_multiple_files = true; +option java_outer_classname = "QuaternionProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// A quaternion is defined as the quotient of two directed lines in a +// three-dimensional space or equivalently as the quotient of two Euclidean +// vectors (https://en.wikipedia.org/wiki/Quaternion). +// +// Quaternions are often used in calculations involving three-dimensional +// rotations (https://en.wikipedia.org/wiki/Quaternions_and_spatial_rotation), +// as they provide greater mathematical robustness by avoiding the gimbal lock +// problems that can be encountered when using Euler angles +// (https://en.wikipedia.org/wiki/Gimbal_lock). +// +// Quaternions are generally represented in this form: +// +// w + xi + yj + zk +// +// where x, y, z, and w are real numbers, and i, j, and k are three imaginary +// numbers. +// +// Our naming choice `(x, y, z, w)` comes from the desire to avoid confusion for +// those interested in the geometric properties of the quaternion in the 3D +// Cartesian space. Other texts often use alternative names or subscripts, such +// as `(a, b, c, d)`, `(1, i, j, k)`, or `(0, 1, 2, 3)`, which are perhaps +// better suited for mathematical interpretations. +// +// To avoid any confusion, as well as to maintain compatibility with a large +// number of software libraries, the quaternions represented using the protocol +// buffer below *must* follow the Hamilton convention, which defines `ij = k` +// (i.e. a right-handed algebra), and therefore: +// +// i^2 = j^2 = k^2 = ijk = −1 +// ij = −ji = k +// jk = −kj = i +// ki = −ik = j +// +// Please DO NOT use this to represent quaternions that follow the JPL +// convention, or any of the other quaternion flavors out there. +// +// Definitions: +// +// - Quaternion norm (or magnitude): `sqrt(x^2 + y^2 + z^2 + w^2)`. +// - Unit (or normalized) quaternion: a quaternion whose norm is 1. +// - Pure quaternion: a quaternion whose scalar component (`w`) is 0. +// - Rotation quaternion: a unit quaternion used to represent rotation. +// - Orientation quaternion: a unit quaternion used to represent orientation. +// +// A quaternion can be normalized by dividing it by its norm. The resulting +// quaternion maintains the same direction, but has a norm of 1, i.e. it moves +// on the unit sphere. This is generally necessary for rotation and orientation +// quaternions, to avoid rounding errors: +// https://en.wikipedia.org/wiki/Rotation_formalisms_in_three_dimensions +// +// Note that `(x, y, z, w)` and `(-x, -y, -z, -w)` represent the same rotation, +// but normalization would be even more useful, e.g. for comparison purposes, if +// it would produce a unique representation. It is thus recommended that `w` be +// kept positive, which can be achieved by changing all the signs when `w` is +// negative. +// +message Quaternion { + // The x component. + double x = 1; + + // The y component. + double y = 2; + + // The z component. + double z = 3; + + // The scalar component. + double w = 4; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/timeofday.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/timeofday.proto new file mode 100644 index 000000000000..5cb48aa936f3 --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/timeofday.proto @@ -0,0 +1,44 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/timeofday;timeofday"; +option java_multiple_files = true; +option java_outer_classname = "TimeOfDayProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a time of day. The date and time zone are either not significant +// or are specified elsewhere. An API may choose to allow leap seconds. Related +// types are [google.type.Date][google.type.Date] and +// `google.protobuf.Timestamp`. +message TimeOfDay { + // Hours of day in 24 hour format. Should be from 0 to 23. An API may choose + // to allow the value "24:00:00" for scenarios like business closing time. + int32 hours = 1; + + // Minutes of hour of day. Must be from 0 to 59. + int32 minutes = 2; + + // Seconds of minutes of the time. Must normally be from 0 to 59. An API may + // allow the value 60 if it allows leap-seconds. + int32 seconds = 3; + + // Fractions of seconds in nanoseconds. Must be from 0 to 999,999,999. + int32 nanos = 4; +} diff --git a/src/go/rpk/pkg/serde/proto.go b/src/go/rpk/pkg/serde/proto.go index 3604b9b20a6a..93c0f17c7829 100644 --- a/src/go/rpk/pkg/serde/proto.go +++ b/src/go/rpk/pkg/serde/proto.go @@ -16,6 +16,7 @@ import ( "github.com/bufbuild/protocompile" "github.com/bufbuild/protocompile/linker" + "github.com/redpanda-data/redpanda/src/go/rpk/pkg/serde/embed" "github.com/twmb/franz-go/pkg/sr" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" @@ -154,18 +155,32 @@ func compileSchema(ctx context.Context, cl *sr.Client, schema *sr.Schema) (linke // This is the original schema. accessorMap[inMemFileName] = schema.Schema - // And we add the rest of schemas if we have references. + // Add Schema References. if len(schema.References) > 0 { err := mapReferences(ctx, cl, schema, accessorMap) if err != nil { return nil, fmt.Errorf("unable to map references: %v", err) } } - + // Redpanda includes another set of common protobuf types in + // https://github.com/redpanda-data/redpanda/tree/dev/src/v/pandaproxy/schema_registry/protobuf + // We embed it into our source accessor map to keep parity with Redpanda. + commonMap, err := embed.CommonProtoFileMap() + if err != nil { + return nil, fmt.Errorf("unable to get common protobuf types: %v", err) + } + for commonPath, commonSchema := range commonMap { + if _, exists := accessorMap[commonPath]; !exists { + accessorMap[commonPath] = commonSchema + } + } + resolver := &protocompile.SourceResolver{ + Accessor: protocompile.SourceAccessorFromMap(accessorMap), + } compiler := protocompile.Compiler{ - Resolver: &protocompile.SourceResolver{ - Accessor: protocompile.SourceAccessorFromMap(accessorMap), - }, + // And we finally add a set of well-known protobuf types: + // https://github.com/bufbuild/protocompile/blob/main/std_imports.go#L42. + Resolver: protocompile.WithStandardImports(resolver), SourceInfoMode: protocompile.SourceInfoStandard, } compiled, err := compiler.Compile(ctx, inMemFileName) diff --git a/src/go/rpk/pkg/serde/proto_test.go b/src/go/rpk/pkg/serde/proto_test.go index 2b344e901898..3b1934680d87 100644 --- a/src/go/rpk/pkg/serde/proto_test.go +++ b/src/go/rpk/pkg/serde/proto_test.go @@ -105,6 +105,95 @@ message Person { .Person.PhoneNumber phone = 2; }` + // https://github.com/redpanda-data/redpanda/blob/dev/src/v/pandaproxy/schema_registry/test/compatibility_protobuf.cc + const testWellKnownSchema = `syntax = "proto3"; +package test; +import "google/protobuf/any.proto"; +import "google/protobuf/api.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/empty.proto"; +import "google/protobuf/field_mask.proto"; +import "google/protobuf/source_context.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/timestamp.proto"; +import "google/protobuf/type.proto"; +import "google/protobuf/wrappers.proto"; +import "google/type/calendar_period.proto"; +import "google/type/color.proto"; +import "google/type/date.proto"; +import "google/type/datetime.proto"; +import "google/type/dayofweek.proto"; +import "google/type/decimal.proto"; +import "google/type/expr.proto"; +import "google/type/fraction.proto"; +import "google/type/interval.proto"; +import "google/type/latlng.proto"; +import "google/type/localized_text.proto"; +import "google/type/money.proto"; +import "google/type/month.proto"; +import "google/type/phone_number.proto"; +import "google/type/postal_address.proto"; +import "google/type/quaternion.proto"; +import "google/type/timeofday.proto"; +import "confluent/meta.proto"; +import "confluent/types/decimal.proto"; + +message well_known_types { + google.protobuf.Any any = 1; + google.protobuf.Api api = 2; + google.protobuf.BoolValue bool_value = 3; + google.protobuf.BytesValue bytes_value = 4; + google.protobuf.DoubleValue double_value = 5; + google.protobuf.Duration duration = 6; + google.protobuf.Empty empty = 7; + google.protobuf.Enum enum = 8; + google.protobuf.EnumValue enum_value = 9; + google.protobuf.Field field = 10; + google.protobuf.FieldMask field_mask = 11; + google.protobuf.FloatValue float_value = 12; + google.protobuf.Int32Value int32_value = 13; + google.protobuf.Int64Value int64_value = 14; + google.protobuf.ListValue list_value = 15; + google.protobuf.Method method = 16; + google.protobuf.Mixin mixin = 17; + google.protobuf.NullValue null_value = 18; + google.protobuf.Option option = 19; + google.protobuf.SourceContext source_context = 20; + google.protobuf.StringValue string_value = 21; + google.protobuf.Struct struct = 22; + google.protobuf.Syntax syntax = 23; + google.protobuf.Timestamp timestamp = 24; + google.protobuf.Type type = 25; + google.protobuf.UInt32Value uint32_value = 26; + google.protobuf.UInt64Value uint64_value = 27; + google.protobuf.Value value = 28; + google.type.CalendarPeriod calendar_period = 29; + google.type.Color color = 30; + google.type.Date date = 31; + google.type.DateTime date_time = 32; + google.type.DayOfWeek day_of_week = 33; + google.type.Decimal decimal = 34; + google.type.Expr expr = 35; + google.type.Fraction fraction = 36; + google.type.Interval interval = 37; + google.type.LatLng lat_lng = 39; + google.type.LocalizedText localized_text = 40; + google.type.Money money = 41; + google.type.Month month = 42; + google.type.PhoneNumber phone_number = 43; + google.type.PostalAddress postal_address = 44; + google.type.Quaternion quaternion = 45; + google.type.TimeOfDay time_of_day = 46; + confluent.Meta c_meta = 47; + confluent.type.Decimal c_decimal = 48; +} + +message Person { + string first_name = 1; + string last_name = 2; +} +` + tests := []struct { name string schema string @@ -224,6 +313,14 @@ message Person { record: `{"brand":"pandaPhone","year":2023}`, expRecord: `{"brand":"pandaPhone","year":2023}`, expIdx: []int{0, 1, 1}, + }, { + name: "wellknown - All Fields", + schema: testWellKnownSchema, + msgType: "test.well_known_types", + schemaID: 11, + record: messageAllWellKnown, + expRecord: messageAllWellKnown, + expIdx: []int{0}, }, } for _, tt := range tests { @@ -454,3 +551,168 @@ func protoReferenceHandler() http.HandlerFunc { } } } + +// Long message, including all well-known types baked in rpk that we can +// encode/decode. +const messageAllWellKnown = `{ + "any": { + "@type": "type.googleapis.com/test.Person", + "firstName": "foo", + "lastName": "bar" + }, + "api": { + "version": "v1", + "methods": [ + { + "name": "GetMethod", + "requestTypeUrl": "type.googleapis.com/google.protobuf.Empty", + "responseTypeUrl": "type.googleapis.com/google.protobuf.StringValue" + } + ] + }, + "boolValue": true, + "bytesValue": "bGVasG6gd11ybGQ=", + "doubleValue": 3.14159, + "duration": "3600s", + "empty": {}, + "enum": { + "name": "ZERO" + }, + "enumValue": { + "name": "MY_ENUM_VALUE", + "number": 1 + }, + "field": { + "name": "fieldName" + }, + "fieldMask": "field", + "floatValue": 1.23, + "int32Value": 42, + "int64Value": "123456789", + "listValue": [ + { "stringValue": "Item 1" }, + { "int32Value": 100 } + ], + "method": { + "name": "MethodName", + "requestTypeUrl": "type.googleapis.com/google.protobuf.StringValue", + "responseTypeUrl": "type.googleapis.com/google.protobuf.StringValue" + }, + "mixin": { + "name": "MixinName", + "root": "rootPath" + }, + "option": { + "name": "optionName", + "value": { + "@type": "type.googleapis.com/test.Person", + "firstName": "foo", + "lastName": "bar" + } + }, + "sourceContext": { + "fileName": "fileName.proto" + }, + "stringValue": "This is a string", + "struct": { + "fields": { + "field1": { "stringValue": "value1" } + } + }, + "syntax": "SYNTAX_PROTO3", + "timestamp": "2020-05-22T20:32:05Z", + "type": { + "name": "TypeName", + "fields": [ + { "name": "field1", "typeUrl": "type.googleapis.com/google.protobuf.StringValue" } + ] + }, + "uint32Value": 9876521, + "uint64Value": "9876543210", + "value": { + "stringValue": "A value example" + }, + "calendarPeriod": "DAY", + "color": { + "red": 255, + "green": 100, + "blue": 50, + "alpha": 0.8 + }, + "date": { + "year": 2024, + "month": 12, + "day": 5 + }, + "dateTime": { + "year": 2024, + "month": 12, + "day": 5, + "hours": 14, + "minutes": 30, + "seconds": 15, + "nanos": 123456789, + "utcOffset": "3600s" + }, + "dayOfWeek": "WEDNESDAY", + "decimal": { + "value": "123.456" + }, + "expr": { + "expression": "a + b", + "title": "Sample Expression" + }, + "fraction": { + "numerator": "3", + "denominator": "4" + }, + "interval": { + "startTime": "2020-05-22T20:32:05Z", + "endTime": "2023-01-01T20:32:05Z" + }, + "latLng": { + "latitude": 37.7749, + "longitude": -122.4194 + }, + "localizedText": { + "text": "Hello", + "languageCode": "en-US" + }, + "money": { + "currencyCode": "USD", + "units": "100", + "nanos": 99 + }, + "month": "DECEMBER", + "phoneNumber": { + "e164Number": "+15552220123", + "extension": "1234" + }, + "postalAddress": { + "regionCode": "US", + "postalCode": "94105", + "locality": "San Francisco", + "addressLines": ["123 Main St", "Suite 456"] + }, + "quaternion": { + "x": 1.0, + "y": 1.0, + "z": 1.0, + "w": 1.0 + }, + "timeOfDay": { + "hours": 14, + "minutes": 30, + "seconds": 1, + "nanos": 2 + }, + "cMeta": { + "doc": "v2" + }, + "cDecimal": { + "value": "AQAAAG9wI1Xh", + "precision": 10, + "scale": 2 + } +} +` diff --git a/src/v/pandaproxy/BUILD b/src/v/pandaproxy/BUILD index 65eeaa5002b2..5f523b5fc0c7 100644 --- a/src/v/pandaproxy/BUILD +++ b/src/v/pandaproxy/BUILD @@ -35,6 +35,7 @@ proto_library( "schema_registry/protobuf/google/type/quaternion.proto", "schema_registry/protobuf/google/type/timeofday.proto", ], + visibility = ["//src/go/rpk/pkg/serde:__subpackages__"], deps = [ "@protobuf//:any_proto", "@protobuf//:descriptor_proto", diff --git a/tests/rptest/clients/rpk.py b/tests/rptest/clients/rpk.py index bad49f3cf930..a57c14ccb99f 100644 --- a/tests/rptest/clients/rpk.py +++ b/tests/rptest/clients/rpk.py @@ -1624,11 +1624,7 @@ def _schema_registry_conn_settings(self): ] return flags - def _run_registry(self, - cmd, - stdin=None, - timeout=None, - output_format="json"): + def _run_registry(self, cmd, stdin=None, timeout=60, output_format="json"): cmd = [self._rpk_binary(), "registry", "--format", output_format ] + self._schema_registry_conn_settings() + cmd out = self._execute(cmd, stdin=stdin, timeout=timeout) diff --git a/tests/rptest/tests/rpk_registry_test.py b/tests/rptest/tests/rpk_registry_test.py index ce9028831ef6..bf9c7818e103 100644 --- a/tests/rptest/tests/rpk_registry_test.py +++ b/tests/rptest/tests/rpk_registry_test.py @@ -11,10 +11,12 @@ import socket import json +from ducktape.utils.util import wait_until from rptest.services.cluster import cluster from rptest.tests.redpanda_test import RedpandaTest from rptest.services.redpanda import SchemaRegistryConfig, SecurityConfig from rptest.clients.rpk import RpkTool, RpkException +from rptest.services.admin import Admin from rptest.services import tls from rptest.tests.pandaproxy_test import User, PandaProxyTLSProvider from rptest.util import expect_exception @@ -51,6 +53,7 @@ def __init__(self, ctx, schema_registry_config=SchemaRegistryConfig()): super(RpkRegistryTest, self).__init__( test_context=ctx, schema_registry_config=schema_registry_config, + node_ready_timeout_s=60, ) # SASL Config self.security = SecurityConfig() @@ -86,7 +89,6 @@ def setUp(self): self.schema_registry_config.client_crt = self.admin_user.certificate.crt self.redpanda.set_security_settings(self.security) self.redpanda.set_schema_registry_settings(self.schema_registry_config) - self.redpanda.start() self._rpk = RpkTool(self.redpanda, @@ -100,6 +102,24 @@ def setUp(self): self.admin_user.password, self.admin_user.algorithm) + # wait for users to propagate to nodes + admin = Admin(self.redpanda) + + def auth_user_propagated(): + for node in self.redpanda.nodes: + users = admin.list_users(node=node) + if self.admin_user.username not in users: + return False + return True + + wait_until(auth_user_propagated, timeout_sec=60, backoff_sec=3) + + # Wait until Schema Registry is ready. + wait_until(self._schema_topic_created, + timeout_sec=60, + backoff_sec=3, + retry_on_exc=True) + def create_schema(self, subject, schema, suffix, references=None): with tempfile.NamedTemporaryFile(suffix=suffix) as tf: tf.write(bytes(schema, 'UTF-8')) @@ -109,6 +129,11 @@ def create_schema(self, subject, schema, suffix, references=None): references=references) assert out["subject"] == subject + def _schema_topic_created(self): + # SR should create the topic the first time its accessed. + self._rpk.list_schemas() + return "_schemas" in self._rpk.list_topics() + @cluster(num_nodes=3) def test_registry_schema(self): subject_1 = "test_subject_1" @@ -361,6 +386,7 @@ def test_produce_consume_avro(self): @cluster(num_nodes=3) def test_produce_consume_proto(self): + # First we register the schemas with their references. subject_1 = "subject_for_person" proto_person = """ @@ -448,6 +474,53 @@ def test_produce_consume_proto(self): assert json.loads(msg["value"]) == expected_msg_1 assert json.loads(msg["key"]) == expected_msg_2 + # Testing the same as above but using a well-known protobuf type + subject_3 = "subject_for_well_known_types" + well_known_proto_def = """ +syntax = "proto3"; + +import "google/protobuf/timestamp.proto"; +import "google/type/month.proto"; + +message Test3 { + google.protobuf.Timestamp timestamp = 1; + google.type.Month month = 2; +} +""" + self.create_schema(subject_3, well_known_proto_def, ".proto") # ID 3 + + msg_3 = '{"timestamp":"2024-12-18T20:32:05Z","month":"DECEMBER"}' + key_3 = "somekey" + expected_msg_3 = json.loads(msg_3) + # Produce: unencoded key, encoded value: + self._rpk.produce( + test_topic, msg=msg_3, key=key_3, schema_id=3, + proto_msg="") # For single-message, it should default to it. + + # We consume as is, i.e: it will show the encoded value. + out = self._rpk.consume(test_topic, offset="2:3") + msg = json.loads(out) + + raw_bytes_string = msg["value"] + assert raw_bytes_string != expected_msg_3, f'expected to have raw bytes {raw_bytes_string} to be different than {expected_msg_3}' + assert msg["key"] == key_3, f'got key: {msg["key"]}; expected {key_3}' + + bytes_from_string = bytes( + raw_bytes_string.encode().decode('unicode-escape'), 'utf-8') + assert bytes_from_string[ + 0] == 0, f'expected encoding to contain magic byte (0)' + + # Now we decode the same message: + out = self._rpk.consume(test_topic, + offset="2:3", + use_schema_registry="value") + msg = json.loads(out) + + assert json.loads( + msg["value"] + ) == expected_msg_3, f'got: {json.loads(msg["value"])}; expected {expected_msg_3}' + assert msg["key"] == key_3, f'got key: {msg["key"]}; expected {key_3}' + @cluster(num_nodes=3) def test_produce_consume_json(self): # First we register the schemas with their references.