Skip to content

Commit

Permalink
rpk: add well-known protobuf types to rpk
Browse files Browse the repository at this point in the history
This is the same set of well-known types that
both Redpanda and Redpanda Console already support

This allows users to use the whole suite and
encode/decode using these well-known types.
  • Loading branch information
r-vasquez committed Dec 9, 2024
1 parent cddb5ef commit 99fa280
Show file tree
Hide file tree
Showing 28 changed files with 1,786 additions and 5 deletions.
1 change: 1 addition & 0 deletions src/go/rpk/pkg/serde/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ go_library(
importpath = "github.com/redpanda-data/redpanda/src/go/rpk/pkg/serde",
visibility = ["//visibility:public"],
deps = [
"//src/go/rpk/pkg/serde/embed",
"@com_github_bufbuild_protocompile//:protocompile",
"@com_github_bufbuild_protocompile//linker",
"@com_github_hamba_avro_v2//:avro",
Expand Down
38 changes: 38 additions & 0 deletions src/go/rpk/pkg/serde/embed/BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
load("@rules_go//go:def.bzl", "go_library", "go_test")

go_library(
name = "embed",
srcs = ["embed.go"],
embedsrcs = [
"protobuf/confluent/meta.proto",
"protobuf/confluent/types/decimal.proto",
"protobuf/google/type/README.md",
"protobuf/google/type/calendar_period.proto",
"protobuf/google/type/color.proto",
"protobuf/google/type/date.proto",
"protobuf/google/type/datetime.proto",
"protobuf/google/type/dayofweek.proto",
"protobuf/google/type/decimal.proto",
"protobuf/google/type/expr.proto",
"protobuf/google/type/fraction.proto",
"protobuf/google/type/interval.proto",
"protobuf/google/type/latlng.proto",
"protobuf/google/type/localized_text.proto",
"protobuf/google/type/money.proto",
"protobuf/google/type/month.proto",
"protobuf/google/type/phone_number.proto",
"protobuf/google/type/postal_address.proto",
"protobuf/google/type/quaternion.proto",
"protobuf/google/type/timeofday.proto",
"protobuf/.clang-format",
],
importpath = "github.com/redpanda-data/redpanda/src/go/rpk/pkg/serde/embed",
visibility = ["//visibility:public"],
)

go_test(
name = "embed_test",
srcs = ["embed_test.go"],
embed = [":embed"],
deps = ["@com_github_stretchr_testify//require"],
)
64 changes: 64 additions & 0 deletions src/go/rpk/pkg/serde/embed/embed.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
// Copyright 2024 Redpanda Data, Inc.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.md
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0

package embed

import (
"embed"
"io/fs"
"path/filepath"
"sync"
)

//go:embed protobuf/google/type/*.proto protobuf/confluent/*.proto protobuf/confluent/types/*.proto
var content embed.FS

var (
once sync.Once
protoMap map[string]string
)

// CommonProtoFiles returns the file system representation of the common
// protobuf types.
func CommonProtoFiles() (fs.FS, error) {
return fs.Sub(content, "protobuf")
}

// CommonProtoFileMap returns the map representation of the common protobuf
// types. This is useful for protoreflect parsing.
func CommonProtoFileMap() (map[string]string, error) {
protoFS, err := CommonProtoFiles()
if err != nil {
return nil, err
}

once.Do(func() {
protoMap = make(map[string]string)
err = fs.WalkDir(protoFS, ".", func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
if filepath.Ext(path) == ".proto" {
data, err := fs.ReadFile(protoFS, path)
if err == nil {
protoMap[path] = string(data)
}
}
return nil
})
})

if err != nil {
return nil, err
}
return protoMap, err
}
44 changes: 44 additions & 0 deletions src/go/rpk/pkg/serde/embed/embed_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package embed

import (
"io/fs"
"os"
"path/filepath"
"testing"

"github.com/stretchr/testify/require"
)

func TestEmbeddedFiles(t *testing.T) {
t.Run("Test Embedded files in rpk, equal to Redpanda", func(t *testing.T) {
// /src/v/pandaproxy/schema_registry/protobuf
redpandaProtoFS := os.DirFS("../../../../../v/pandaproxy/schema_registry/protobuf/")
redpandaMap := make(map[string]string)
err := fs.WalkDir(redpandaProtoFS, ".", func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
if filepath.Ext(path) == ".proto" {
data, err := fs.ReadFile(redpandaProtoFS, path)
if err == nil {
redpandaMap[path] = string(data)
}
}
return nil
})

embeddedMap, err := CommonProtoFileMap()
require.NoError(t, err)

for path, embedContent := range embeddedMap {
if rpContent, ok := redpandaMap[path]; ok {
require.Equalf(t, rpContent, embedContent, "Contents of %v have changed vs the embedded rpk files", path)
} else {
t.Fatalf("%s not found in Redpanda files", path)
}
}
})
}
2 changes: 2 additions & 0 deletions src/go/rpk/pkg/serde/embed/protobuf/.clang-format
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Language: Proto
DisableFormat: true
27 changes: 27 additions & 0 deletions src/go/rpk/pkg/serde/embed/protobuf/confluent/meta.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
syntax = "proto3";
package confluent;

import "google/protobuf/descriptor.proto";

option go_package="../confluent";

message Meta {
string doc = 1;
map<string, string> params = 2;
}

extend google.protobuf.FileOptions {
Meta file_meta = 1088;
}
extend google.protobuf.MessageOptions {
Meta message_meta = 1088;
}
extend google.protobuf.FieldOptions {
Meta field_meta = 1088;
}
extend google.protobuf.EnumOptions {
Meta enum_meta = 1088;
}
extend google.protobuf.EnumValueOptions {
Meta enum_value_meta = 1088;
}
17 changes: 17 additions & 0 deletions src/go/rpk/pkg/serde/embed/protobuf/confluent/types/decimal.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
syntax = "proto3";

package confluent.type;

option go_package="../types";

message Decimal {

// The two's-complement representation of the unscaled integer value in big-endian byte order
bytes value = 1;

// The precision
uint32 precision = 2;

// The scale
int32 scale = 3;
}
16 changes: 16 additions & 0 deletions src/go/rpk/pkg/serde/embed/protobuf/google/type/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# Google Common Types

This package contains definitions of common types for Google APIs.
All types defined in this package are suitable for different APIs to
exchange data, and will never break binary compatibility. They should
have design quality comparable to major programming languages like
Java and C#.

NOTE: Some common types are defined in the package `google.protobuf`
as they are directly supported by Protocol Buffers compiler and
runtime. Those types are called Well-Known Types.

## Java Utilities

A set of Java utilities for the Common Types are provided in the
`//java/com/google/type/util/` package.
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package google.type;

option go_package = "google.golang.org/genproto/googleapis/type/calendarperiod;calendarperiod";
option java_multiple_files = true;
option java_outer_classname = "CalendarPeriodProto";
option java_package = "com.google.type";
option objc_class_prefix = "GTP";

// A `CalendarPeriod` represents the abstract concept of a time period that has
// a canonical start. Grammatically, "the start of the current
// `CalendarPeriod`." All calendar times begin at midnight UTC.
enum CalendarPeriod {
// Undefined period, raises an error.
CALENDAR_PERIOD_UNSPECIFIED = 0;

// A day.
DAY = 1;

// A week. Weeks begin on Monday, following
// [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date).
WEEK = 2;

// A fortnight. The first calendar fortnight of the year begins at the start
// of week 1 according to
// [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date).
FORTNIGHT = 3;

// A month.
MONTH = 4;

// A quarter. Quarters start on dates 1-Jan, 1-Apr, 1-Jul, and 1-Oct of each
// year.
QUARTER = 5;

// A half-year. Half-years start on dates 1-Jan and 1-Jul.
HALF = 6;

// A year.
YEAR = 7;
}
Loading

0 comments on commit 99fa280

Please sign in to comment.