-
Notifications
You must be signed in to change notification settings - Fork 592
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
rpk: add well-known protobuf types to rpk
This is the same set of well-known types that both Redpanda and Redpanda Console already support This allows users to use the whole suite and encode/decode using these well-known types.
- Loading branch information
Showing
28 changed files
with
1,786 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,38 @@ | ||
load("@rules_go//go:def.bzl", "go_library", "go_test") | ||
|
||
go_library( | ||
name = "embed", | ||
srcs = ["embed.go"], | ||
embedsrcs = [ | ||
"protobuf/confluent/meta.proto", | ||
"protobuf/confluent/types/decimal.proto", | ||
"protobuf/google/type/README.md", | ||
"protobuf/google/type/calendar_period.proto", | ||
"protobuf/google/type/color.proto", | ||
"protobuf/google/type/date.proto", | ||
"protobuf/google/type/datetime.proto", | ||
"protobuf/google/type/dayofweek.proto", | ||
"protobuf/google/type/decimal.proto", | ||
"protobuf/google/type/expr.proto", | ||
"protobuf/google/type/fraction.proto", | ||
"protobuf/google/type/interval.proto", | ||
"protobuf/google/type/latlng.proto", | ||
"protobuf/google/type/localized_text.proto", | ||
"protobuf/google/type/money.proto", | ||
"protobuf/google/type/month.proto", | ||
"protobuf/google/type/phone_number.proto", | ||
"protobuf/google/type/postal_address.proto", | ||
"protobuf/google/type/quaternion.proto", | ||
"protobuf/google/type/timeofday.proto", | ||
"protobuf/.clang-format", | ||
], | ||
importpath = "github.com/redpanda-data/redpanda/src/go/rpk/pkg/serde/embed", | ||
visibility = ["//visibility:public"], | ||
) | ||
|
||
go_test( | ||
name = "embed_test", | ||
srcs = ["embed_test.go"], | ||
embed = [":embed"], | ||
deps = ["@com_github_stretchr_testify//require"], | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,64 @@ | ||
// Copyright 2024 Redpanda Data, Inc. | ||
// | ||
// Use of this software is governed by the Business Source License | ||
// included in the file licenses/BSL.md | ||
// | ||
// As of the Change Date specified in that file, in accordance with | ||
// the Business Source License, use of this software will be governed | ||
// by the Apache License, Version 2.0 | ||
|
||
package embed | ||
|
||
import ( | ||
"embed" | ||
"io/fs" | ||
"path/filepath" | ||
"sync" | ||
) | ||
|
||
//go:embed protobuf/google/type/*.proto protobuf/confluent/*.proto protobuf/confluent/types/*.proto | ||
var content embed.FS | ||
|
||
var ( | ||
once sync.Once | ||
protoMap map[string]string | ||
) | ||
|
||
// CommonProtoFiles returns the file system representation of the common | ||
// protobuf types. | ||
func CommonProtoFiles() (fs.FS, error) { | ||
return fs.Sub(content, "protobuf") | ||
} | ||
|
||
// CommonProtoFileMap returns the map representation of the common protobuf | ||
// types. This is useful for protoreflect parsing. | ||
func CommonProtoFileMap() (map[string]string, error) { | ||
protoFS, err := CommonProtoFiles() | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
once.Do(func() { | ||
protoMap = make(map[string]string) | ||
err = fs.WalkDir(protoFS, ".", func(path string, d fs.DirEntry, err error) error { | ||
if err != nil { | ||
return err | ||
} | ||
if d.IsDir() { | ||
return nil | ||
} | ||
if filepath.Ext(path) == ".proto" { | ||
data, err := fs.ReadFile(protoFS, path) | ||
if err == nil { | ||
protoMap[path] = string(data) | ||
} | ||
} | ||
return nil | ||
}) | ||
}) | ||
|
||
if err != nil { | ||
return nil, err | ||
} | ||
return protoMap, err | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,44 @@ | ||
package embed | ||
|
||
import ( | ||
"io/fs" | ||
"os" | ||
"path/filepath" | ||
"testing" | ||
|
||
"github.com/stretchr/testify/require" | ||
) | ||
|
||
func TestEmbeddedFiles(t *testing.T) { | ||
t.Run("Test Embedded files in rpk, equal to Redpanda", func(t *testing.T) { | ||
// /src/v/pandaproxy/schema_registry/protobuf | ||
redpandaProtoFS := os.DirFS("../../../../../v/pandaproxy/schema_registry/protobuf/") | ||
redpandaMap := make(map[string]string) | ||
err := fs.WalkDir(redpandaProtoFS, ".", func(path string, d fs.DirEntry, err error) error { | ||
if err != nil { | ||
return err | ||
} | ||
if d.IsDir() { | ||
return nil | ||
} | ||
if filepath.Ext(path) == ".proto" { | ||
data, err := fs.ReadFile(redpandaProtoFS, path) | ||
if err == nil { | ||
redpandaMap[path] = string(data) | ||
} | ||
} | ||
return nil | ||
}) | ||
|
||
embeddedMap, err := CommonProtoFileMap() | ||
require.NoError(t, err) | ||
|
||
for path, embedContent := range embeddedMap { | ||
if rpContent, ok := redpandaMap[path]; ok { | ||
require.Equalf(t, rpContent, embedContent, "Contents of %v have changed vs the embedded rpk files", path) | ||
} else { | ||
t.Fatalf("%s not found in Redpanda files", path) | ||
} | ||
} | ||
}) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
Language: Proto | ||
DisableFormat: true |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
syntax = "proto3"; | ||
package confluent; | ||
|
||
import "google/protobuf/descriptor.proto"; | ||
|
||
option go_package="../confluent"; | ||
|
||
message Meta { | ||
string doc = 1; | ||
map<string, string> params = 2; | ||
} | ||
|
||
extend google.protobuf.FileOptions { | ||
Meta file_meta = 1088; | ||
} | ||
extend google.protobuf.MessageOptions { | ||
Meta message_meta = 1088; | ||
} | ||
extend google.protobuf.FieldOptions { | ||
Meta field_meta = 1088; | ||
} | ||
extend google.protobuf.EnumOptions { | ||
Meta enum_meta = 1088; | ||
} | ||
extend google.protobuf.EnumValueOptions { | ||
Meta enum_value_meta = 1088; | ||
} |
17 changes: 17 additions & 0 deletions
17
src/go/rpk/pkg/serde/embed/protobuf/confluent/types/decimal.proto
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
syntax = "proto3"; | ||
|
||
package confluent.type; | ||
|
||
option go_package="../types"; | ||
|
||
message Decimal { | ||
|
||
// The two's-complement representation of the unscaled integer value in big-endian byte order | ||
bytes value = 1; | ||
|
||
// The precision | ||
uint32 precision = 2; | ||
|
||
// The scale | ||
int32 scale = 3; | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
# Google Common Types | ||
|
||
This package contains definitions of common types for Google APIs. | ||
All types defined in this package are suitable for different APIs to | ||
exchange data, and will never break binary compatibility. They should | ||
have design quality comparable to major programming languages like | ||
Java and C#. | ||
|
||
NOTE: Some common types are defined in the package `google.protobuf` | ||
as they are directly supported by Protocol Buffers compiler and | ||
runtime. Those types are called Well-Known Types. | ||
|
||
## Java Utilities | ||
|
||
A set of Java utilities for the Common Types are provided in the | ||
`//java/com/google/type/util/` package. |
56 changes: 56 additions & 0 deletions
56
src/go/rpk/pkg/serde/embed/protobuf/google/type/calendar_period.proto
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,56 @@ | ||
// Copyright 2021 Google LLC | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
syntax = "proto3"; | ||
|
||
package google.type; | ||
|
||
option go_package = "google.golang.org/genproto/googleapis/type/calendarperiod;calendarperiod"; | ||
option java_multiple_files = true; | ||
option java_outer_classname = "CalendarPeriodProto"; | ||
option java_package = "com.google.type"; | ||
option objc_class_prefix = "GTP"; | ||
|
||
// A `CalendarPeriod` represents the abstract concept of a time period that has | ||
// a canonical start. Grammatically, "the start of the current | ||
// `CalendarPeriod`." All calendar times begin at midnight UTC. | ||
enum CalendarPeriod { | ||
// Undefined period, raises an error. | ||
CALENDAR_PERIOD_UNSPECIFIED = 0; | ||
|
||
// A day. | ||
DAY = 1; | ||
|
||
// A week. Weeks begin on Monday, following | ||
// [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date). | ||
WEEK = 2; | ||
|
||
// A fortnight. The first calendar fortnight of the year begins at the start | ||
// of week 1 according to | ||
// [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date). | ||
FORTNIGHT = 3; | ||
|
||
// A month. | ||
MONTH = 4; | ||
|
||
// A quarter. Quarters start on dates 1-Jan, 1-Apr, 1-Jul, and 1-Oct of each | ||
// year. | ||
QUARTER = 5; | ||
|
||
// A half-year. Half-years start on dates 1-Jan and 1-Jul. | ||
HALF = 6; | ||
|
||
// A year. | ||
YEAR = 7; | ||
} |
Oops, something went wrong.