diff --git a/.github/workflows/rpk-build.yml b/.github/workflows/rpk-build.yml index b762a90a4991..1b42e4113b09 100644 --- a/.github/workflows/rpk-build.yml +++ b/.github/workflows/rpk-build.yml @@ -13,10 +13,12 @@ on: branches: [dev] paths: - 'src/go/rpk/**' + - 'src/v/pandaproxy/schema_registry/protobuf/**' - '.github/workflows/rpk-build.yml' pull_request: paths: - 'src/go/rpk/**' + - 'src/v/pandaproxy/schema_registry/protobuf/**' - '.github/workflows/rpk-build.yml' jobs: test: diff --git a/src/go/rpk/pkg/serde/BUILD b/src/go/rpk/pkg/serde/BUILD index 240f96c40e00..a7fb6b167a98 100644 --- a/src/go/rpk/pkg/serde/BUILD +++ b/src/go/rpk/pkg/serde/BUILD @@ -11,6 +11,7 @@ go_library( importpath = "github.com/redpanda-data/redpanda/src/go/rpk/pkg/serde", visibility = ["//visibility:public"], deps = [ + "//src/go/rpk/pkg/serde/embed", "@com_github_bufbuild_protocompile//:protocompile", "@com_github_bufbuild_protocompile//linker", "@com_github_hamba_avro_v2//:avro", diff --git a/src/go/rpk/pkg/serde/embed/BUILD b/src/go/rpk/pkg/serde/embed/BUILD new file mode 100644 index 000000000000..303033b8187c --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/BUILD @@ -0,0 +1,38 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "embed", + srcs = ["embed.go"], + embedsrcs = [ + "protobuf/confluent/meta.proto", + "protobuf/confluent/types/decimal.proto", + "protobuf/google/type/calendar_period.proto", + "protobuf/google/type/color.proto", + "protobuf/google/type/date.proto", + "protobuf/google/type/datetime.proto", + "protobuf/google/type/dayofweek.proto", + "protobuf/google/type/decimal.proto", + "protobuf/google/type/expr.proto", + "protobuf/google/type/fraction.proto", + "protobuf/google/type/interval.proto", + "protobuf/google/type/latlng.proto", + "protobuf/google/type/localized_text.proto", + "protobuf/google/type/money.proto", + "protobuf/google/type/month.proto", + "protobuf/google/type/phone_number.proto", + "protobuf/google/type/postal_address.proto", + "protobuf/google/type/quaternion.proto", + "protobuf/google/type/timeofday.proto", + ], + importpath = "github.com/redpanda-data/redpanda/src/go/rpk/pkg/serde/embed", + visibility = ["//visibility:public"], +) + +go_test( + name = "embed_test", + size = "small", + srcs = ["embed_test.go"], + data = ["//src/v/pandaproxy:schema_registry_protos"], + embed = [":embed"], + deps = ["@com_github_stretchr_testify//require"], +) diff --git a/src/go/rpk/pkg/serde/embed/embed.go b/src/go/rpk/pkg/serde/embed/embed.go new file mode 100644 index 000000000000..07d7bd1ba502 --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/embed.go @@ -0,0 +1,62 @@ +// Copyright 2024 Redpanda Data, Inc. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.md +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0 + +package embed + +import ( + "embed" + "io/fs" + "path/filepath" + "sync" +) + +//go:embed protobuf/google/type/*.proto protobuf/confluent/*.proto protobuf/confluent/types/*.proto +var content embed.FS + +var ( + once sync.Once + protoMap map[string]string +) + +// CommonProtoFiles returns the file system representation of the common +// protobuf types. +func CommonProtoFiles() (fs.FS, error) { + return fs.Sub(content, "protobuf") +} + +// CommonProtoFileMap returns the map representation of the common protobuf +// types. This is useful for protoreflect parsing. +func CommonProtoFileMap() (map[string]string, error) { + protoFS, err := CommonProtoFiles() + if err != nil { + return nil, err + } + + once.Do(func() { + protoMap = make(map[string]string) + err = fs.WalkDir(protoFS, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || filepath.Ext(path) != ".proto" { + return nil + } + data, err := fs.ReadFile(protoFS, path) + if err == nil { + protoMap[path] = string(data) + } + return nil + }) + }) + + if err != nil { + return nil, err + } + return protoMap, err +} diff --git a/src/go/rpk/pkg/serde/embed/embed_test.go b/src/go/rpk/pkg/serde/embed/embed_test.go new file mode 100644 index 000000000000..e96c728f3d0b --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/embed_test.go @@ -0,0 +1,42 @@ +package embed + +import ( + "io/fs" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestEmbeddedFiles(t *testing.T) { + t.Run("Test Embedded files in rpk, equal to Redpanda", func(t *testing.T) { + // /src/v/pandaproxy/schema_registry/protobuf + redpandaProtoFS := os.DirFS("../../../../../v/pandaproxy/schema_registry/protobuf/") + redpandaMap := make(map[string]string) + err := fs.WalkDir(redpandaProtoFS, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || filepath.Ext(path) != ".proto" { + return nil + } + data, err := fs.ReadFile(redpandaProtoFS, path) + if err == nil { + redpandaMap[path] = string(data) + } + return nil + }) + + embeddedMap, err := CommonProtoFileMap() + require.NoError(t, err) + + for path, embedContent := range embeddedMap { + if rpContent, ok := redpandaMap[path]; ok { + require.Equalf(t, rpContent, embedContent, "Contents of %v have changed vs the embedded rpk files", path) + } else { + t.Fatalf("%s not found in Redpanda files", path) + } + } + }) +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/.clang-format b/src/go/rpk/pkg/serde/embed/protobuf/.clang-format new file mode 100644 index 000000000000..ef9566ac8a7a --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/.clang-format @@ -0,0 +1,2 @@ +Language: Proto +DisableFormat: true diff --git a/src/go/rpk/pkg/serde/embed/protobuf/confluent/meta.proto b/src/go/rpk/pkg/serde/embed/protobuf/confluent/meta.proto new file mode 100644 index 000000000000..4439139d0c68 --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/confluent/meta.proto @@ -0,0 +1,27 @@ +syntax = "proto3"; +package confluent; + +import "google/protobuf/descriptor.proto"; + +option go_package="../confluent"; + +message Meta { + string doc = 1; + map params = 2; +} + +extend google.protobuf.FileOptions { + Meta file_meta = 1088; +} +extend google.protobuf.MessageOptions { + Meta message_meta = 1088; +} +extend google.protobuf.FieldOptions { + Meta field_meta = 1088; +} +extend google.protobuf.EnumOptions { + Meta enum_meta = 1088; +} +extend google.protobuf.EnumValueOptions { + Meta enum_value_meta = 1088; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/confluent/types/decimal.proto b/src/go/rpk/pkg/serde/embed/protobuf/confluent/types/decimal.proto new file mode 100644 index 000000000000..75d8b9b46f0a --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/confluent/types/decimal.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +package confluent.type; + +option go_package="../types"; + +message Decimal { + + // The two's-complement representation of the unscaled integer value in big-endian byte order + bytes value = 1; + + // The precision + uint32 precision = 2; + + // The scale + int32 scale = 3; +} \ No newline at end of file diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/README.md b/src/go/rpk/pkg/serde/embed/protobuf/google/type/README.md new file mode 100644 index 000000000000..6caf02cf1f5e --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/README.md @@ -0,0 +1,16 @@ +# Google Common Types + +This package contains definitions of common types for Google APIs. +All types defined in this package are suitable for different APIs to +exchange data, and will never break binary compatibility. They should +have design quality comparable to major programming languages like +Java and C#. + +NOTE: Some common types are defined in the package `google.protobuf` +as they are directly supported by Protocol Buffers compiler and +runtime. Those types are called Well-Known Types. + +## Java Utilities + +A set of Java utilities for the Common Types are provided in the +`//java/com/google/type/util/` package. \ No newline at end of file diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/calendar_period.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/calendar_period.proto new file mode 100644 index 000000000000..82f5690b752b --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/calendar_period.proto @@ -0,0 +1,56 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/calendarperiod;calendarperiod"; +option java_multiple_files = true; +option java_outer_classname = "CalendarPeriodProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// A `CalendarPeriod` represents the abstract concept of a time period that has +// a canonical start. Grammatically, "the start of the current +// `CalendarPeriod`." All calendar times begin at midnight UTC. +enum CalendarPeriod { + // Undefined period, raises an error. + CALENDAR_PERIOD_UNSPECIFIED = 0; + + // A day. + DAY = 1; + + // A week. Weeks begin on Monday, following + // [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date). + WEEK = 2; + + // A fortnight. The first calendar fortnight of the year begins at the start + // of week 1 according to + // [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date). + FORTNIGHT = 3; + + // A month. + MONTH = 4; + + // A quarter. Quarters start on dates 1-Jan, 1-Apr, 1-Jul, and 1-Oct of each + // year. + QUARTER = 5; + + // A half-year. Half-years start on dates 1-Jan and 1-Jul. + HALF = 6; + + // A year. + YEAR = 7; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/color.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/color.proto new file mode 100644 index 000000000000..5dc85a6a3856 --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/color.proto @@ -0,0 +1,174 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +import "google/protobuf/wrappers.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/color;color"; +option java_multiple_files = true; +option java_outer_classname = "ColorProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a color in the RGBA color space. This representation is designed +// for simplicity of conversion to/from color representations in various +// languages over compactness. For example, the fields of this representation +// can be trivially provided to the constructor of `java.awt.Color` in Java; it +// can also be trivially provided to UIColor's `+colorWithRed:green:blue:alpha` +// method in iOS; and, with just a little work, it can be easily formatted into +// a CSS `rgba()` string in JavaScript. +// +// This reference page doesn't carry information about the absolute color +// space +// that should be used to interpret the RGB value (e.g. sRGB, Adobe RGB, +// DCI-P3, BT.2020, etc.). By default, applications should assume the sRGB color +// space. +// +// When color equality needs to be decided, implementations, unless +// documented otherwise, treat two colors as equal if all their red, +// green, blue, and alpha values each differ by at most 1e-5. +// +// Example (Java): +// +// import com.google.type.Color; +// +// // ... +// public static java.awt.Color fromProto(Color protocolor) { +// float alpha = protocolor.hasAlpha() +// ? protocolor.getAlpha().getValue() +// : 1.0; +// +// return new java.awt.Color( +// protocolor.getRed(), +// protocolor.getGreen(), +// protocolor.getBlue(), +// alpha); +// } +// +// public static Color toProto(java.awt.Color color) { +// float red = (float) color.getRed(); +// float green = (float) color.getGreen(); +// float blue = (float) color.getBlue(); +// float denominator = 255.0; +// Color.Builder resultBuilder = +// Color +// .newBuilder() +// .setRed(red / denominator) +// .setGreen(green / denominator) +// .setBlue(blue / denominator); +// int alpha = color.getAlpha(); +// if (alpha != 255) { +// result.setAlpha( +// FloatValue +// .newBuilder() +// .setValue(((float) alpha) / denominator) +// .build()); +// } +// return resultBuilder.build(); +// } +// // ... +// +// Example (iOS / Obj-C): +// +// // ... +// static UIColor* fromProto(Color* protocolor) { +// float red = [protocolor red]; +// float green = [protocolor green]; +// float blue = [protocolor blue]; +// FloatValue* alpha_wrapper = [protocolor alpha]; +// float alpha = 1.0; +// if (alpha_wrapper != nil) { +// alpha = [alpha_wrapper value]; +// } +// return [UIColor colorWithRed:red green:green blue:blue alpha:alpha]; +// } +// +// static Color* toProto(UIColor* color) { +// CGFloat red, green, blue, alpha; +// if (![color getRed:&red green:&green blue:&blue alpha:&alpha]) { +// return nil; +// } +// Color* result = [[Color alloc] init]; +// [result setRed:red]; +// [result setGreen:green]; +// [result setBlue:blue]; +// if (alpha <= 0.9999) { +// [result setAlpha:floatWrapperWithValue(alpha)]; +// } +// [result autorelease]; +// return result; +// } +// // ... +// +// Example (JavaScript): +// +// // ... +// +// var protoToCssColor = function(rgb_color) { +// var redFrac = rgb_color.red || 0.0; +// var greenFrac = rgb_color.green || 0.0; +// var blueFrac = rgb_color.blue || 0.0; +// var red = Math.floor(redFrac * 255); +// var green = Math.floor(greenFrac * 255); +// var blue = Math.floor(blueFrac * 255); +// +// if (!('alpha' in rgb_color)) { +// return rgbToCssColor(red, green, blue); +// } +// +// var alphaFrac = rgb_color.alpha.value || 0.0; +// var rgbParams = [red, green, blue].join(','); +// return ['rgba(', rgbParams, ',', alphaFrac, ')'].join(''); +// }; +// +// var rgbToCssColor = function(red, green, blue) { +// var rgbNumber = new Number((red << 16) | (green << 8) | blue); +// var hexString = rgbNumber.toString(16); +// var missingZeros = 6 - hexString.length; +// var resultBuilder = ['#']; +// for (var i = 0; i < missingZeros; i++) { +// resultBuilder.push('0'); +// } +// resultBuilder.push(hexString); +// return resultBuilder.join(''); +// }; +// +// // ... +message Color { + // The amount of red in the color as a value in the interval [0, 1]. + float red = 1; + + // The amount of green in the color as a value in the interval [0, 1]. + float green = 2; + + // The amount of blue in the color as a value in the interval [0, 1]. + float blue = 3; + + // The fraction of this color that should be applied to the pixel. That is, + // the final pixel color is defined by the equation: + // + // `pixel color = alpha * (this color) + (1.0 - alpha) * (background color)` + // + // This means that a value of 1.0 corresponds to a solid color, whereas + // a value of 0.0 corresponds to a completely transparent color. This + // uses a wrapper message rather than a simple float scalar so that it is + // possible to distinguish between a default value and the value being unset. + // If omitted, this color object is rendered as a solid color + // (as if the alpha value had been explicitly given a value of 1.0). + google.protobuf.FloatValue alpha = 4; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/date.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/date.proto new file mode 100644 index 000000000000..e4e730e6f5a9 --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/date.proto @@ -0,0 +1,52 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/date;date"; +option java_multiple_files = true; +option java_outer_classname = "DateProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a whole or partial calendar date, such as a birthday. The time of +// day and time zone are either specified elsewhere or are insignificant. The +// date is relative to the Gregorian Calendar. This can represent one of the +// following: +// +// * A full date, with non-zero year, month, and day values +// * A month and day value, with a zero year, such as an anniversary +// * A year on its own, with zero month and day values +// * A year and month value, with a zero day, such as a credit card expiration +// date +// +// Related types are [google.type.TimeOfDay][google.type.TimeOfDay] and +// `google.protobuf.Timestamp`. +message Date { + // Year of the date. Must be from 1 to 9999, or 0 to specify a date without + // a year. + int32 year = 1; + + // Month of a year. Must be from 1 to 12, or 0 to specify a year without a + // month and day. + int32 month = 2; + + // Day of a month. Must be from 1 to 31 and valid for the year and month, or 0 + // to specify a year by itself or a year and month where the day isn't + // significant. + int32 day = 3; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/datetime.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/datetime.proto new file mode 100644 index 000000000000..cfed85d70a10 --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/datetime.proto @@ -0,0 +1,104 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +import "google/protobuf/duration.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/datetime;datetime"; +option java_multiple_files = true; +option java_outer_classname = "DateTimeProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents civil time (or occasionally physical time). +// +// This type can represent a civil time in one of a few possible ways: +// +// * When utc_offset is set and time_zone is unset: a civil time on a calendar +// day with a particular offset from UTC. +// * When time_zone is set and utc_offset is unset: a civil time on a calendar +// day in a particular time zone. +// * When neither time_zone nor utc_offset is set: a civil time on a calendar +// day in local time. +// +// The date is relative to the Proleptic Gregorian Calendar. +// +// If year is 0, the DateTime is considered not to have a specific year. month +// and day must have valid, non-zero values. +// +// This type may also be used to represent a physical time if all the date and +// time fields are set and either case of the `time_offset` oneof is set. +// Consider using `Timestamp` message for physical time instead. If your use +// case also would like to store the user's timezone, that can be done in +// another field. +// +// This type is more flexible than some applications may want. Make sure to +// document and validate your application's limitations. +message DateTime { + // Optional. Year of date. Must be from 1 to 9999, or 0 if specifying a + // datetime without a year. + int32 year = 1; + + // Required. Month of year. Must be from 1 to 12. + int32 month = 2; + + // Required. Day of month. Must be from 1 to 31 and valid for the year and + // month. + int32 day = 3; + + // Required. Hours of day in 24 hour format. Should be from 0 to 23. An API + // may choose to allow the value "24:00:00" for scenarios like business + // closing time. + int32 hours = 4; + + // Required. Minutes of hour of day. Must be from 0 to 59. + int32 minutes = 5; + + // Required. Seconds of minutes of the time. Must normally be from 0 to 59. An + // API may allow the value 60 if it allows leap-seconds. + int32 seconds = 6; + + // Required. Fractions of seconds in nanoseconds. Must be from 0 to + // 999,999,999. + int32 nanos = 7; + + // Optional. Specifies either the UTC offset or the time zone of the DateTime. + // Choose carefully between them, considering that time zone data may change + // in the future (for example, a country modifies their DST start/end dates, + // and future DateTimes in the affected range had already been stored). + // If omitted, the DateTime is considered to be in local time. + oneof time_offset { + // UTC offset. Must be whole seconds, between -18 hours and +18 hours. + // For example, a UTC offset of -4:00 would be represented as + // { seconds: -14400 }. + google.protobuf.Duration utc_offset = 8; + + // Time zone. + TimeZone time_zone = 9; + } +} + +// Represents a time zone from the +// [IANA Time Zone Database](https://www.iana.org/time-zones). +message TimeZone { + // IANA Time Zone Database time zone, e.g. "America/New_York". + string id = 1; + + // Optional. IANA Time Zone Database version number, e.g. "2019a". + string version = 2; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/dayofweek.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/dayofweek.proto new file mode 100644 index 000000000000..4c80c62ec0b4 --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/dayofweek.proto @@ -0,0 +1,50 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/dayofweek;dayofweek"; +option java_multiple_files = true; +option java_outer_classname = "DayOfWeekProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a day of the week. +enum DayOfWeek { + // The day of the week is unspecified. + DAY_OF_WEEK_UNSPECIFIED = 0; + + // Monday + MONDAY = 1; + + // Tuesday + TUESDAY = 2; + + // Wednesday + WEDNESDAY = 3; + + // Thursday + THURSDAY = 4; + + // Friday + FRIDAY = 5; + + // Saturday + SATURDAY = 6; + + // Sunday + SUNDAY = 7; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/decimal.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/decimal.proto new file mode 100644 index 000000000000..beb18a5d8dd2 --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/decimal.proto @@ -0,0 +1,95 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/decimal;decimal"; +option java_multiple_files = true; +option java_outer_classname = "DecimalProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// A representation of a decimal value, such as 2.5. Clients may convert values +// into language-native decimal formats, such as Java's [BigDecimal][] or +// Python's [decimal.Decimal][]. +// +// [BigDecimal]: +// https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/math/BigDecimal.html +// [decimal.Decimal]: https://docs.python.org/3/library/decimal.html +message Decimal { + // The decimal value, as a string. + // + // The string representation consists of an optional sign, `+` (`U+002B`) + // or `-` (`U+002D`), followed by a sequence of zero or more decimal digits + // ("the integer"), optionally followed by a fraction, optionally followed + // by an exponent. + // + // The fraction consists of a decimal point followed by zero or more decimal + // digits. The string must contain at least one digit in either the integer + // or the fraction. The number formed by the sign, the integer and the + // fraction is referred to as the significand. + // + // The exponent consists of the character `e` (`U+0065`) or `E` (`U+0045`) + // followed by one or more decimal digits. + // + // Services **should** normalize decimal values before storing them by: + // + // - Removing an explicitly-provided `+` sign (`+2.5` -> `2.5`). + // - Replacing a zero-length integer value with `0` (`.5` -> `0.5`). + // - Coercing the exponent character to lower-case (`2.5E8` -> `2.5e8`). + // - Removing an explicitly-provided zero exponent (`2.5e0` -> `2.5`). + // + // Services **may** perform additional normalization based on its own needs + // and the internal decimal implementation selected, such as shifting the + // decimal point and exponent value together (example: `2.5e-1` <-> `0.25`). + // Additionally, services **may** preserve trailing zeroes in the fraction + // to indicate increased precision, but are not required to do so. + // + // Note that only the `.` character is supported to divide the integer + // and the fraction; `,` **should not** be supported regardless of locale. + // Additionally, thousand separators **should not** be supported. If a + // service does support them, values **must** be normalized. + // + // The ENBF grammar is: + // + // DecimalString = + // [Sign] Significand [Exponent]; + // + // Sign = '+' | '-'; + // + // Significand = + // Digits ['.'] [Digits] | [Digits] '.' Digits; + // + // Exponent = ('e' | 'E') [Sign] Digits; + // + // Digits = { '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' }; + // + // Services **should** clearly document the range of supported values, the + // maximum supported precision (total number of digits), and, if applicable, + // the scale (number of digits after the decimal point), as well as how it + // behaves when receiving out-of-bounds values. + // + // Services **may** choose to accept values passed as input even when the + // value has a higher precision or scale than the service supports, and + // **should** round the value to fit the supported scale. Alternatively, the + // service **may** error with `400 Bad Request` (`INVALID_ARGUMENT` in gRPC) + // if precision would be lost. + // + // Services **should** error with `400 Bad Request` (`INVALID_ARGUMENT` in + // gRPC) if the service receives a value outside of the supported range. + string value = 1; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/expr.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/expr.proto new file mode 100644 index 000000000000..af0778cf958c --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/expr.proto @@ -0,0 +1,73 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/expr;expr"; +option java_multiple_files = true; +option java_outer_classname = "ExprProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a textual expression in the Common Expression Language (CEL) +// syntax. CEL is a C-like expression language. The syntax and semantics of CEL +// are documented at https://github.com/google/cel-spec. +// +// Example (Comparison): +// +// title: "Summary size limit" +// description: "Determines if a summary is less than 100 chars" +// expression: "document.summary.size() < 100" +// +// Example (Equality): +// +// title: "Requestor is owner" +// description: "Determines if requestor is the document owner" +// expression: "document.owner == request.auth.claims.email" +// +// Example (Logic): +// +// title: "Public documents" +// description: "Determine whether the document should be publicly visible" +// expression: "document.type != 'private' && document.type != 'internal'" +// +// Example (Data Manipulation): +// +// title: "Notification string" +// description: "Create a notification string with a timestamp." +// expression: "'New message received at ' + string(document.create_time)" +// +// The exact variables and functions that may be referenced within an expression +// are determined by the service that evaluates it. See the service +// documentation for additional information. +message Expr { + // Textual representation of an expression in Common Expression Language + // syntax. + string expression = 1; + + // Optional. Title for the expression, i.e. a short string describing + // its purpose. This can be used e.g. in UIs which allow to enter the + // expression. + string title = 2; + + // Optional. Description of the expression. This is a longer text which + // describes the expression, e.g. when hovered over it in a UI. + string description = 3; + + // Optional. String indicating the location of the expression for error + // reporting, e.g. a file name and a position in the file. + string location = 4; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/fraction.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/fraction.proto new file mode 100644 index 000000000000..6c5ae6e2a25d --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/fraction.proto @@ -0,0 +1,33 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/fraction;fraction"; +option java_multiple_files = true; +option java_outer_classname = "FractionProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a fraction in terms of a numerator divided by a denominator. +message Fraction { + // The numerator in the fraction, e.g. 2 in 2/3. + int64 numerator = 1; + + // The value by which the numerator is divided, e.g. 3 in 2/3. Must be + // positive. + int64 denominator = 2; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/interval.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/interval.proto new file mode 100644 index 000000000000..9702324cd4e8 --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/interval.proto @@ -0,0 +1,46 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +import "google/protobuf/timestamp.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/interval;interval"; +option java_multiple_files = true; +option java_outer_classname = "IntervalProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a time interval, encoded as a Timestamp start (inclusive) and a +// Timestamp end (exclusive). +// +// The start must be less than or equal to the end. +// When the start equals the end, the interval is empty (matches no time). +// When both start and end are unspecified, the interval matches any time. +message Interval { + // Optional. Inclusive start of the interval. + // + // If specified, a Timestamp matching this interval will have to be the same + // or after the start. + google.protobuf.Timestamp start_time = 1; + + // Optional. Exclusive end of the interval. + // + // If specified, a Timestamp matching this interval will have to be before the + // end. + google.protobuf.Timestamp end_time = 2; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/latlng.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/latlng.proto new file mode 100644 index 000000000000..9231456e328f --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/latlng.proto @@ -0,0 +1,37 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/latlng;latlng"; +option java_multiple_files = true; +option java_outer_classname = "LatLngProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// An object that represents a latitude/longitude pair. This is expressed as a +// pair of doubles to represent degrees latitude and degrees longitude. Unless +// specified otherwise, this must conform to the +// WGS84 +// standard. Values must be within normalized ranges. +message LatLng { + // The latitude in degrees. It must be in the range [-90.0, +90.0]. + double latitude = 1; + + // The longitude in degrees. It must be in the range [-180.0, +180.0]. + double longitude = 2; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/localized_text.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/localized_text.proto new file mode 100644 index 000000000000..5c6922b8c0cd --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/localized_text.proto @@ -0,0 +1,36 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/localized_text;localized_text"; +option java_multiple_files = true; +option java_outer_classname = "LocalizedTextProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Localized variant of a text in a particular language. +message LocalizedText { + // Localized string in the language corresponding to `language_code' below. + string text = 1; + + // The text's BCP-47 language code, such as "en-US" or "sr-Latn". + // + // For more information, see + // http://www.unicode.org/reports/tr35/#Unicode_locale_identifier. + string language_code = 2; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/money.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/money.proto new file mode 100644 index 000000000000..98d6494e4210 --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/money.proto @@ -0,0 +1,42 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/money;money"; +option java_multiple_files = true; +option java_outer_classname = "MoneyProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents an amount of money with its currency type. +message Money { + // The three-letter currency code defined in ISO 4217. + string currency_code = 1; + + // The whole units of the amount. + // For example if `currencyCode` is `"USD"`, then 1 unit is one US dollar. + int64 units = 2; + + // Number of nano (10^-9) units of the amount. + // The value must be between -999,999,999 and +999,999,999 inclusive. + // If `units` is positive, `nanos` must be positive or zero. + // If `units` is zero, `nanos` can be positive, zero, or negative. + // If `units` is negative, `nanos` must be negative or zero. + // For example $-1.75 is represented as `units`=-1 and `nanos`=-750,000,000. + int32 nanos = 3; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/month.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/month.proto new file mode 100644 index 000000000000..99e7551b1416 --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/month.proto @@ -0,0 +1,65 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/month;month"; +option java_multiple_files = true; +option java_outer_classname = "MonthProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a month in the Gregorian calendar. +enum Month { + // The unspecified month. + MONTH_UNSPECIFIED = 0; + + // The month of January. + JANUARY = 1; + + // The month of February. + FEBRUARY = 2; + + // The month of March. + MARCH = 3; + + // The month of April. + APRIL = 4; + + // The month of May. + MAY = 5; + + // The month of June. + JUNE = 6; + + // The month of July. + JULY = 7; + + // The month of August. + AUGUST = 8; + + // The month of September. + SEPTEMBER = 9; + + // The month of October. + OCTOBER = 10; + + // The month of November. + NOVEMBER = 11; + + // The month of December. + DECEMBER = 12; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/phone_number.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/phone_number.proto new file mode 100644 index 000000000000..7bbb7d873229 --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/phone_number.proto @@ -0,0 +1,113 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/phone_number;phone_number"; +option java_multiple_files = true; +option java_outer_classname = "PhoneNumberProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// An object representing a phone number, suitable as an API wire format. +// +// This representation: +// +// - should not be used for locale-specific formatting of a phone number, such +// as "+1 (650) 253-0000 ext. 123" +// +// - is not designed for efficient storage +// - may not be suitable for dialing - specialized libraries (see references) +// should be used to parse the number for that purpose +// +// To do something meaningful with this number, such as format it for various +// use-cases, convert it to an `i18n.phonenumbers.PhoneNumber` object first. +// +// For instance, in Java this would be: +// +// com.google.type.PhoneNumber wireProto = +// com.google.type.PhoneNumber.newBuilder().build(); +// com.google.i18n.phonenumbers.Phonenumber.PhoneNumber phoneNumber = +// PhoneNumberUtil.getInstance().parse(wireProto.getE164Number(), "ZZ"); +// if (!wireProto.getExtension().isEmpty()) { +// phoneNumber.setExtension(wireProto.getExtension()); +// } +// +// Reference(s): +// - https://github.com/google/libphonenumber +message PhoneNumber { + // An object representing a short code, which is a phone number that is + // typically much shorter than regular phone numbers and can be used to + // address messages in MMS and SMS systems, as well as for abbreviated dialing + // (e.g. "Text 611 to see how many minutes you have remaining on your plan."). + // + // Short codes are restricted to a region and are not internationally + // dialable, which means the same short code can exist in different regions, + // with different usage and pricing, even if those regions share the same + // country calling code (e.g. US and CA). + message ShortCode { + // Required. The BCP-47 region code of the location where calls to this + // short code can be made, such as "US" and "BB". + // + // Reference(s): + // - http://www.unicode.org/reports/tr35/#unicode_region_subtag + string region_code = 1; + + // Required. The short code digits, without a leading plus ('+') or country + // calling code, e.g. "611". + string number = 2; + } + + // Required. Either a regular number, or a short code. New fields may be + // added to the oneof below in the future, so clients should ignore phone + // numbers for which none of the fields they coded against are set. + oneof kind { + // The phone number, represented as a leading plus sign ('+'), followed by a + // phone number that uses a relaxed ITU E.164 format consisting of the + // country calling code (1 to 3 digits) and the subscriber number, with no + // additional spaces or formatting, e.g.: + // - correct: "+15552220123" + // - incorrect: "+1 (555) 222-01234 x123". + // + // The ITU E.164 format limits the latter to 12 digits, but in practice not + // all countries respect that, so we relax that restriction here. + // National-only numbers are not allowed. + // + // References: + // - https://www.itu.int/rec/T-REC-E.164-201011-I + // - https://en.wikipedia.org/wiki/E.164. + // - https://en.wikipedia.org/wiki/List_of_country_calling_codes + string e164_number = 1; + + // A short code. + // + // Reference(s): + // - https://en.wikipedia.org/wiki/Short_code + ShortCode short_code = 2; + } + + // The phone number's extension. The extension is not standardized in ITU + // recommendations, except for being defined as a series of numbers with a + // maximum length of 40 digits. Other than digits, some other dialing + // characters such as ',' (indicating a wait) or '#' may be stored here. + // + // Note that no regions currently use extensions with short codes, so this + // field is normally only set in conjunction with an E.164 number. It is held + // separately from the E.164 number to allow for short code extensions in the + // future. + string extension = 3; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/postal_address.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/postal_address.proto new file mode 100644 index 000000000000..c57c7c31a2cc --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/postal_address.proto @@ -0,0 +1,134 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/postaladdress;postaladdress"; +option java_multiple_files = true; +option java_outer_classname = "PostalAddressProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a postal address, e.g. for postal delivery or payments addresses. +// Given a postal address, a postal service can deliver items to a premise, P.O. +// Box or similar. +// It is not intended to model geographical locations (roads, towns, +// mountains). +// +// In typical usage an address would be created via user input or from importing +// existing data, depending on the type of process. +// +// Advice on address input / editing: +// - Use an i18n-ready address widget such as +// https://github.com/google/libaddressinput) +// - Users should not be presented with UI elements for input or editing of +// fields outside countries where that field is used. +// +// For more guidance on how to use this schema, please see: +// https://support.google.com/business/answer/6397478 +message PostalAddress { + // The schema revision of the `PostalAddress`. This must be set to 0, which is + // the latest revision. + // + // All new revisions **must** be backward compatible with old revisions. + int32 revision = 1; + + // Required. CLDR region code of the country/region of the address. This + // is never inferred and it is up to the user to ensure the value is + // correct. See http://cldr.unicode.org/ and + // http://www.unicode.org/cldr/charts/30/supplemental/territory_information.html + // for details. Example: "CH" for Switzerland. + string region_code = 2; + + // Optional. BCP-47 language code of the contents of this address (if + // known). This is often the UI language of the input form or is expected + // to match one of the languages used in the address' country/region, or their + // transliterated equivalents. + // This can affect formatting in certain countries, but is not critical + // to the correctness of the data and will never affect any validation or + // other non-formatting related operations. + // + // If this value is not known, it should be omitted (rather than specifying a + // possibly incorrect default). + // + // Examples: "zh-Hant", "ja", "ja-Latn", "en". + string language_code = 3; + + // Optional. Postal code of the address. Not all countries use or require + // postal codes to be present, but where they are used, they may trigger + // additional validation with other parts of the address (e.g. state/zip + // validation in the U.S.A.). + string postal_code = 4; + + // Optional. Additional, country-specific, sorting code. This is not used + // in most regions. Where it is used, the value is either a string like + // "CEDEX", optionally followed by a number (e.g. "CEDEX 7"), or just a number + // alone, representing the "sector code" (Jamaica), "delivery area indicator" + // (Malawi) or "post office indicator" (e.g. Côte d'Ivoire). + string sorting_code = 5; + + // Optional. Highest administrative subdivision which is used for postal + // addresses of a country or region. + // For example, this can be a state, a province, an oblast, or a prefecture. + // Specifically, for Spain this is the province and not the autonomous + // community (e.g. "Barcelona" and not "Catalonia"). + // Many countries don't use an administrative area in postal addresses. E.g. + // in Switzerland this should be left unpopulated. + string administrative_area = 6; + + // Optional. Generally refers to the city/town portion of the address. + // Examples: US city, IT comune, UK post town. + // In regions of the world where localities are not well defined or do not fit + // into this structure well, leave locality empty and use address_lines. + string locality = 7; + + // Optional. Sublocality of the address. + // For example, this can be neighborhoods, boroughs, districts. + string sublocality = 8; + + // Unstructured address lines describing the lower levels of an address. + // + // Because values in address_lines do not have type information and may + // sometimes contain multiple values in a single field (e.g. + // "Austin, TX"), it is important that the line order is clear. The order of + // address lines should be "envelope order" for the country/region of the + // address. In places where this can vary (e.g. Japan), address_language is + // used to make it explicit (e.g. "ja" for large-to-small ordering and + // "ja-Latn" or "en" for small-to-large). This way, the most specific line of + // an address can be selected based on the language. + // + // The minimum permitted structural representation of an address consists + // of a region_code with all remaining information placed in the + // address_lines. It would be possible to format such an address very + // approximately without geocoding, but no semantic reasoning could be + // made about any of the address components until it was at least + // partially resolved. + // + // Creating an address only containing a region_code and address_lines, and + // then geocoding is the recommended way to handle completely unstructured + // addresses (as opposed to guessing which parts of the address should be + // localities or administrative areas). + repeated string address_lines = 9; + + // Optional. The recipient at the address. + // This field may, under certain circumstances, contain multiline information. + // For example, it might contain "care of" information. + repeated string recipients = 10; + + // Optional. The name of the organization at the address. + string organization = 11; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/quaternion.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/quaternion.proto new file mode 100644 index 000000000000..dfb822deff16 --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/quaternion.proto @@ -0,0 +1,94 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/quaternion;quaternion"; +option java_multiple_files = true; +option java_outer_classname = "QuaternionProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// A quaternion is defined as the quotient of two directed lines in a +// three-dimensional space or equivalently as the quotient of two Euclidean +// vectors (https://en.wikipedia.org/wiki/Quaternion). +// +// Quaternions are often used in calculations involving three-dimensional +// rotations (https://en.wikipedia.org/wiki/Quaternions_and_spatial_rotation), +// as they provide greater mathematical robustness by avoiding the gimbal lock +// problems that can be encountered when using Euler angles +// (https://en.wikipedia.org/wiki/Gimbal_lock). +// +// Quaternions are generally represented in this form: +// +// w + xi + yj + zk +// +// where x, y, z, and w are real numbers, and i, j, and k are three imaginary +// numbers. +// +// Our naming choice `(x, y, z, w)` comes from the desire to avoid confusion for +// those interested in the geometric properties of the quaternion in the 3D +// Cartesian space. Other texts often use alternative names or subscripts, such +// as `(a, b, c, d)`, `(1, i, j, k)`, or `(0, 1, 2, 3)`, which are perhaps +// better suited for mathematical interpretations. +// +// To avoid any confusion, as well as to maintain compatibility with a large +// number of software libraries, the quaternions represented using the protocol +// buffer below *must* follow the Hamilton convention, which defines `ij = k` +// (i.e. a right-handed algebra), and therefore: +// +// i^2 = j^2 = k^2 = ijk = −1 +// ij = −ji = k +// jk = −kj = i +// ki = −ik = j +// +// Please DO NOT use this to represent quaternions that follow the JPL +// convention, or any of the other quaternion flavors out there. +// +// Definitions: +// +// - Quaternion norm (or magnitude): `sqrt(x^2 + y^2 + z^2 + w^2)`. +// - Unit (or normalized) quaternion: a quaternion whose norm is 1. +// - Pure quaternion: a quaternion whose scalar component (`w`) is 0. +// - Rotation quaternion: a unit quaternion used to represent rotation. +// - Orientation quaternion: a unit quaternion used to represent orientation. +// +// A quaternion can be normalized by dividing it by its norm. The resulting +// quaternion maintains the same direction, but has a norm of 1, i.e. it moves +// on the unit sphere. This is generally necessary for rotation and orientation +// quaternions, to avoid rounding errors: +// https://en.wikipedia.org/wiki/Rotation_formalisms_in_three_dimensions +// +// Note that `(x, y, z, w)` and `(-x, -y, -z, -w)` represent the same rotation, +// but normalization would be even more useful, e.g. for comparison purposes, if +// it would produce a unique representation. It is thus recommended that `w` be +// kept positive, which can be achieved by changing all the signs when `w` is +// negative. +// +message Quaternion { + // The x component. + double x = 1; + + // The y component. + double y = 2; + + // The z component. + double z = 3; + + // The scalar component. + double w = 4; +} diff --git a/src/go/rpk/pkg/serde/embed/protobuf/google/type/timeofday.proto b/src/go/rpk/pkg/serde/embed/protobuf/google/type/timeofday.proto new file mode 100644 index 000000000000..5cb48aa936f3 --- /dev/null +++ b/src/go/rpk/pkg/serde/embed/protobuf/google/type/timeofday.proto @@ -0,0 +1,44 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/timeofday;timeofday"; +option java_multiple_files = true; +option java_outer_classname = "TimeOfDayProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a time of day. The date and time zone are either not significant +// or are specified elsewhere. An API may choose to allow leap seconds. Related +// types are [google.type.Date][google.type.Date] and +// `google.protobuf.Timestamp`. +message TimeOfDay { + // Hours of day in 24 hour format. Should be from 0 to 23. An API may choose + // to allow the value "24:00:00" for scenarios like business closing time. + int32 hours = 1; + + // Minutes of hour of day. Must be from 0 to 59. + int32 minutes = 2; + + // Seconds of minutes of the time. Must normally be from 0 to 59. An API may + // allow the value 60 if it allows leap-seconds. + int32 seconds = 3; + + // Fractions of seconds in nanoseconds. Must be from 0 to 999,999,999. + int32 nanos = 4; +} diff --git a/src/go/rpk/pkg/serde/proto.go b/src/go/rpk/pkg/serde/proto.go index 3604b9b20a6a..93c0f17c7829 100644 --- a/src/go/rpk/pkg/serde/proto.go +++ b/src/go/rpk/pkg/serde/proto.go @@ -16,6 +16,7 @@ import ( "github.com/bufbuild/protocompile" "github.com/bufbuild/protocompile/linker" + "github.com/redpanda-data/redpanda/src/go/rpk/pkg/serde/embed" "github.com/twmb/franz-go/pkg/sr" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" @@ -154,18 +155,32 @@ func compileSchema(ctx context.Context, cl *sr.Client, schema *sr.Schema) (linke // This is the original schema. accessorMap[inMemFileName] = schema.Schema - // And we add the rest of schemas if we have references. + // Add Schema References. if len(schema.References) > 0 { err := mapReferences(ctx, cl, schema, accessorMap) if err != nil { return nil, fmt.Errorf("unable to map references: %v", err) } } - + // Redpanda includes another set of common protobuf types in + // https://github.com/redpanda-data/redpanda/tree/dev/src/v/pandaproxy/schema_registry/protobuf + // We embed it into our source accessor map to keep parity with Redpanda. + commonMap, err := embed.CommonProtoFileMap() + if err != nil { + return nil, fmt.Errorf("unable to get common protobuf types: %v", err) + } + for commonPath, commonSchema := range commonMap { + if _, exists := accessorMap[commonPath]; !exists { + accessorMap[commonPath] = commonSchema + } + } + resolver := &protocompile.SourceResolver{ + Accessor: protocompile.SourceAccessorFromMap(accessorMap), + } compiler := protocompile.Compiler{ - Resolver: &protocompile.SourceResolver{ - Accessor: protocompile.SourceAccessorFromMap(accessorMap), - }, + // And we finally add a set of well-known protobuf types: + // https://github.com/bufbuild/protocompile/blob/main/std_imports.go#L42. + Resolver: protocompile.WithStandardImports(resolver), SourceInfoMode: protocompile.SourceInfoStandard, } compiled, err := compiler.Compile(ctx, inMemFileName) diff --git a/src/go/rpk/pkg/serde/proto_test.go b/src/go/rpk/pkg/serde/proto_test.go index 2b344e901898..3b1934680d87 100644 --- a/src/go/rpk/pkg/serde/proto_test.go +++ b/src/go/rpk/pkg/serde/proto_test.go @@ -105,6 +105,95 @@ message Person { .Person.PhoneNumber phone = 2; }` + // https://github.com/redpanda-data/redpanda/blob/dev/src/v/pandaproxy/schema_registry/test/compatibility_protobuf.cc + const testWellKnownSchema = `syntax = "proto3"; +package test; +import "google/protobuf/any.proto"; +import "google/protobuf/api.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/empty.proto"; +import "google/protobuf/field_mask.proto"; +import "google/protobuf/source_context.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/timestamp.proto"; +import "google/protobuf/type.proto"; +import "google/protobuf/wrappers.proto"; +import "google/type/calendar_period.proto"; +import "google/type/color.proto"; +import "google/type/date.proto"; +import "google/type/datetime.proto"; +import "google/type/dayofweek.proto"; +import "google/type/decimal.proto"; +import "google/type/expr.proto"; +import "google/type/fraction.proto"; +import "google/type/interval.proto"; +import "google/type/latlng.proto"; +import "google/type/localized_text.proto"; +import "google/type/money.proto"; +import "google/type/month.proto"; +import "google/type/phone_number.proto"; +import "google/type/postal_address.proto"; +import "google/type/quaternion.proto"; +import "google/type/timeofday.proto"; +import "confluent/meta.proto"; +import "confluent/types/decimal.proto"; + +message well_known_types { + google.protobuf.Any any = 1; + google.protobuf.Api api = 2; + google.protobuf.BoolValue bool_value = 3; + google.protobuf.BytesValue bytes_value = 4; + google.protobuf.DoubleValue double_value = 5; + google.protobuf.Duration duration = 6; + google.protobuf.Empty empty = 7; + google.protobuf.Enum enum = 8; + google.protobuf.EnumValue enum_value = 9; + google.protobuf.Field field = 10; + google.protobuf.FieldMask field_mask = 11; + google.protobuf.FloatValue float_value = 12; + google.protobuf.Int32Value int32_value = 13; + google.protobuf.Int64Value int64_value = 14; + google.protobuf.ListValue list_value = 15; + google.protobuf.Method method = 16; + google.protobuf.Mixin mixin = 17; + google.protobuf.NullValue null_value = 18; + google.protobuf.Option option = 19; + google.protobuf.SourceContext source_context = 20; + google.protobuf.StringValue string_value = 21; + google.protobuf.Struct struct = 22; + google.protobuf.Syntax syntax = 23; + google.protobuf.Timestamp timestamp = 24; + google.protobuf.Type type = 25; + google.protobuf.UInt32Value uint32_value = 26; + google.protobuf.UInt64Value uint64_value = 27; + google.protobuf.Value value = 28; + google.type.CalendarPeriod calendar_period = 29; + google.type.Color color = 30; + google.type.Date date = 31; + google.type.DateTime date_time = 32; + google.type.DayOfWeek day_of_week = 33; + google.type.Decimal decimal = 34; + google.type.Expr expr = 35; + google.type.Fraction fraction = 36; + google.type.Interval interval = 37; + google.type.LatLng lat_lng = 39; + google.type.LocalizedText localized_text = 40; + google.type.Money money = 41; + google.type.Month month = 42; + google.type.PhoneNumber phone_number = 43; + google.type.PostalAddress postal_address = 44; + google.type.Quaternion quaternion = 45; + google.type.TimeOfDay time_of_day = 46; + confluent.Meta c_meta = 47; + confluent.type.Decimal c_decimal = 48; +} + +message Person { + string first_name = 1; + string last_name = 2; +} +` + tests := []struct { name string schema string @@ -224,6 +313,14 @@ message Person { record: `{"brand":"pandaPhone","year":2023}`, expRecord: `{"brand":"pandaPhone","year":2023}`, expIdx: []int{0, 1, 1}, + }, { + name: "wellknown - All Fields", + schema: testWellKnownSchema, + msgType: "test.well_known_types", + schemaID: 11, + record: messageAllWellKnown, + expRecord: messageAllWellKnown, + expIdx: []int{0}, }, } for _, tt := range tests { @@ -454,3 +551,168 @@ func protoReferenceHandler() http.HandlerFunc { } } } + +// Long message, including all well-known types baked in rpk that we can +// encode/decode. +const messageAllWellKnown = `{ + "any": { + "@type": "type.googleapis.com/test.Person", + "firstName": "foo", + "lastName": "bar" + }, + "api": { + "version": "v1", + "methods": [ + { + "name": "GetMethod", + "requestTypeUrl": "type.googleapis.com/google.protobuf.Empty", + "responseTypeUrl": "type.googleapis.com/google.protobuf.StringValue" + } + ] + }, + "boolValue": true, + "bytesValue": "bGVasG6gd11ybGQ=", + "doubleValue": 3.14159, + "duration": "3600s", + "empty": {}, + "enum": { + "name": "ZERO" + }, + "enumValue": { + "name": "MY_ENUM_VALUE", + "number": 1 + }, + "field": { + "name": "fieldName" + }, + "fieldMask": "field", + "floatValue": 1.23, + "int32Value": 42, + "int64Value": "123456789", + "listValue": [ + { "stringValue": "Item 1" }, + { "int32Value": 100 } + ], + "method": { + "name": "MethodName", + "requestTypeUrl": "type.googleapis.com/google.protobuf.StringValue", + "responseTypeUrl": "type.googleapis.com/google.protobuf.StringValue" + }, + "mixin": { + "name": "MixinName", + "root": "rootPath" + }, + "option": { + "name": "optionName", + "value": { + "@type": "type.googleapis.com/test.Person", + "firstName": "foo", + "lastName": "bar" + } + }, + "sourceContext": { + "fileName": "fileName.proto" + }, + "stringValue": "This is a string", + "struct": { + "fields": { + "field1": { "stringValue": "value1" } + } + }, + "syntax": "SYNTAX_PROTO3", + "timestamp": "2020-05-22T20:32:05Z", + "type": { + "name": "TypeName", + "fields": [ + { "name": "field1", "typeUrl": "type.googleapis.com/google.protobuf.StringValue" } + ] + }, + "uint32Value": 9876521, + "uint64Value": "9876543210", + "value": { + "stringValue": "A value example" + }, + "calendarPeriod": "DAY", + "color": { + "red": 255, + "green": 100, + "blue": 50, + "alpha": 0.8 + }, + "date": { + "year": 2024, + "month": 12, + "day": 5 + }, + "dateTime": { + "year": 2024, + "month": 12, + "day": 5, + "hours": 14, + "minutes": 30, + "seconds": 15, + "nanos": 123456789, + "utcOffset": "3600s" + }, + "dayOfWeek": "WEDNESDAY", + "decimal": { + "value": "123.456" + }, + "expr": { + "expression": "a + b", + "title": "Sample Expression" + }, + "fraction": { + "numerator": "3", + "denominator": "4" + }, + "interval": { + "startTime": "2020-05-22T20:32:05Z", + "endTime": "2023-01-01T20:32:05Z" + }, + "latLng": { + "latitude": 37.7749, + "longitude": -122.4194 + }, + "localizedText": { + "text": "Hello", + "languageCode": "en-US" + }, + "money": { + "currencyCode": "USD", + "units": "100", + "nanos": 99 + }, + "month": "DECEMBER", + "phoneNumber": { + "e164Number": "+15552220123", + "extension": "1234" + }, + "postalAddress": { + "regionCode": "US", + "postalCode": "94105", + "locality": "San Francisco", + "addressLines": ["123 Main St", "Suite 456"] + }, + "quaternion": { + "x": 1.0, + "y": 1.0, + "z": 1.0, + "w": 1.0 + }, + "timeOfDay": { + "hours": 14, + "minutes": 30, + "seconds": 1, + "nanos": 2 + }, + "cMeta": { + "doc": "v2" + }, + "cDecimal": { + "value": "AQAAAG9wI1Xh", + "precision": 10, + "scale": 2 + } +} +` diff --git a/src/v/pandaproxy/BUILD b/src/v/pandaproxy/BUILD index 65eeaa5002b2..5f523b5fc0c7 100644 --- a/src/v/pandaproxy/BUILD +++ b/src/v/pandaproxy/BUILD @@ -35,6 +35,7 @@ proto_library( "schema_registry/protobuf/google/type/quaternion.proto", "schema_registry/protobuf/google/type/timeofday.proto", ], + visibility = ["//src/go/rpk/pkg/serde:__subpackages__"], deps = [ "@protobuf//:any_proto", "@protobuf//:descriptor_proto", diff --git a/tests/rptest/clients/rpk.py b/tests/rptest/clients/rpk.py index bad49f3cf930..a57c14ccb99f 100644 --- a/tests/rptest/clients/rpk.py +++ b/tests/rptest/clients/rpk.py @@ -1624,11 +1624,7 @@ def _schema_registry_conn_settings(self): ] return flags - def _run_registry(self, - cmd, - stdin=None, - timeout=None, - output_format="json"): + def _run_registry(self, cmd, stdin=None, timeout=60, output_format="json"): cmd = [self._rpk_binary(), "registry", "--format", output_format ] + self._schema_registry_conn_settings() + cmd out = self._execute(cmd, stdin=stdin, timeout=timeout) diff --git a/tests/rptest/tests/rpk_registry_test.py b/tests/rptest/tests/rpk_registry_test.py index ce9028831ef6..bf9c7818e103 100644 --- a/tests/rptest/tests/rpk_registry_test.py +++ b/tests/rptest/tests/rpk_registry_test.py @@ -11,10 +11,12 @@ import socket import json +from ducktape.utils.util import wait_until from rptest.services.cluster import cluster from rptest.tests.redpanda_test import RedpandaTest from rptest.services.redpanda import SchemaRegistryConfig, SecurityConfig from rptest.clients.rpk import RpkTool, RpkException +from rptest.services.admin import Admin from rptest.services import tls from rptest.tests.pandaproxy_test import User, PandaProxyTLSProvider from rptest.util import expect_exception @@ -51,6 +53,7 @@ def __init__(self, ctx, schema_registry_config=SchemaRegistryConfig()): super(RpkRegistryTest, self).__init__( test_context=ctx, schema_registry_config=schema_registry_config, + node_ready_timeout_s=60, ) # SASL Config self.security = SecurityConfig() @@ -86,7 +89,6 @@ def setUp(self): self.schema_registry_config.client_crt = self.admin_user.certificate.crt self.redpanda.set_security_settings(self.security) self.redpanda.set_schema_registry_settings(self.schema_registry_config) - self.redpanda.start() self._rpk = RpkTool(self.redpanda, @@ -100,6 +102,24 @@ def setUp(self): self.admin_user.password, self.admin_user.algorithm) + # wait for users to propagate to nodes + admin = Admin(self.redpanda) + + def auth_user_propagated(): + for node in self.redpanda.nodes: + users = admin.list_users(node=node) + if self.admin_user.username not in users: + return False + return True + + wait_until(auth_user_propagated, timeout_sec=60, backoff_sec=3) + + # Wait until Schema Registry is ready. + wait_until(self._schema_topic_created, + timeout_sec=60, + backoff_sec=3, + retry_on_exc=True) + def create_schema(self, subject, schema, suffix, references=None): with tempfile.NamedTemporaryFile(suffix=suffix) as tf: tf.write(bytes(schema, 'UTF-8')) @@ -109,6 +129,11 @@ def create_schema(self, subject, schema, suffix, references=None): references=references) assert out["subject"] == subject + def _schema_topic_created(self): + # SR should create the topic the first time its accessed. + self._rpk.list_schemas() + return "_schemas" in self._rpk.list_topics() + @cluster(num_nodes=3) def test_registry_schema(self): subject_1 = "test_subject_1" @@ -361,6 +386,7 @@ def test_produce_consume_avro(self): @cluster(num_nodes=3) def test_produce_consume_proto(self): + # First we register the schemas with their references. subject_1 = "subject_for_person" proto_person = """ @@ -448,6 +474,53 @@ def test_produce_consume_proto(self): assert json.loads(msg["value"]) == expected_msg_1 assert json.loads(msg["key"]) == expected_msg_2 + # Testing the same as above but using a well-known protobuf type + subject_3 = "subject_for_well_known_types" + well_known_proto_def = """ +syntax = "proto3"; + +import "google/protobuf/timestamp.proto"; +import "google/type/month.proto"; + +message Test3 { + google.protobuf.Timestamp timestamp = 1; + google.type.Month month = 2; +} +""" + self.create_schema(subject_3, well_known_proto_def, ".proto") # ID 3 + + msg_3 = '{"timestamp":"2024-12-18T20:32:05Z","month":"DECEMBER"}' + key_3 = "somekey" + expected_msg_3 = json.loads(msg_3) + # Produce: unencoded key, encoded value: + self._rpk.produce( + test_topic, msg=msg_3, key=key_3, schema_id=3, + proto_msg="") # For single-message, it should default to it. + + # We consume as is, i.e: it will show the encoded value. + out = self._rpk.consume(test_topic, offset="2:3") + msg = json.loads(out) + + raw_bytes_string = msg["value"] + assert raw_bytes_string != expected_msg_3, f'expected to have raw bytes {raw_bytes_string} to be different than {expected_msg_3}' + assert msg["key"] == key_3, f'got key: {msg["key"]}; expected {key_3}' + + bytes_from_string = bytes( + raw_bytes_string.encode().decode('unicode-escape'), 'utf-8') + assert bytes_from_string[ + 0] == 0, f'expected encoding to contain magic byte (0)' + + # Now we decode the same message: + out = self._rpk.consume(test_topic, + offset="2:3", + use_schema_registry="value") + msg = json.loads(out) + + assert json.loads( + msg["value"] + ) == expected_msg_3, f'got: {json.loads(msg["value"])}; expected {expected_msg_3}' + assert msg["key"] == key_3, f'got key: {msg["key"]}; expected {key_3}' + @cluster(num_nodes=3) def test_produce_consume_json(self): # First we register the schemas with their references.