Merge pull request #24480 from r-vasquez/well-known-proto

[UX-53] rpk: add well-known protobuf types to rpk

.github/workflows/rpk-build.yml

@@ -13,10 +13,12 @@ on:
     branches: [dev]
     paths:
       - 'src/go/rpk/**'
+      - 'src/v/pandaproxy/schema_registry/protobuf/**'
       - '.github/workflows/rpk-build.yml'
   pull_request:
     paths:
       - 'src/go/rpk/**'
+      - 'src/v/pandaproxy/schema_registry/protobuf/**'
       - '.github/workflows/rpk-build.yml'
 jobs:
   test:

src/go/rpk/pkg/serde/BUILD

@@ -11,6 +11,7 @@ go_library(
     importpath = "github.com/redpanda-data/redpanda/src/go/rpk/pkg/serde",
     visibility = ["//visibility:public"],
     deps = [
+        "//src/go/rpk/pkg/serde/embed",
         "@com_github_bufbuild_protocompile//:protocompile",
         "@com_github_bufbuild_protocompile//linker",
         "@com_github_hamba_avro_v2//:avro",

src/go/rpk/pkg/serde/embed/BUILD

@@ -0,0 +1,38 @@
+load("@rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "embed",
+    srcs = ["embed.go"],
+    embedsrcs = [
+        "protobuf/confluent/meta.proto",
+        "protobuf/confluent/types/decimal.proto",
+        "protobuf/google/type/calendar_period.proto",
+        "protobuf/google/type/color.proto",
+        "protobuf/google/type/date.proto",
+        "protobuf/google/type/datetime.proto",
+        "protobuf/google/type/dayofweek.proto",
+        "protobuf/google/type/decimal.proto",
+        "protobuf/google/type/expr.proto",
+        "protobuf/google/type/fraction.proto",
+        "protobuf/google/type/interval.proto",
+        "protobuf/google/type/latlng.proto",
+        "protobuf/google/type/localized_text.proto",
+        "protobuf/google/type/money.proto",
+        "protobuf/google/type/month.proto",
+        "protobuf/google/type/phone_number.proto",
+        "protobuf/google/type/postal_address.proto",
+        "protobuf/google/type/quaternion.proto",
+        "protobuf/google/type/timeofday.proto",
+    ],
+    importpath = "github.com/redpanda-data/redpanda/src/go/rpk/pkg/serde/embed",
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "embed_test",
+    size = "small",
+    srcs = ["embed_test.go"],
+    data = ["//src/v/pandaproxy:schema_registry_protos"],
+    embed = [":embed"],
+    deps = ["@com_github_stretchr_testify//require"],
+)

src/go/rpk/pkg/serde/embed/embed.go

@@ -0,0 +1,62 @@
+// Copyright 2024 Redpanda Data, Inc.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.md
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0
+
+package embed
+
+import (
+	"embed"
+	"io/fs"
+	"path/filepath"
+	"sync"
+)
+
+//go:embed protobuf/google/type/*.proto protobuf/confluent/*.proto protobuf/confluent/types/*.proto
+var content embed.FS
+
+var (
+	once     sync.Once
+	protoMap map[string]string
+)
+
+// CommonProtoFiles returns the file system representation of the common
+// protobuf types.
+func CommonProtoFiles() (fs.FS, error) {
+	return fs.Sub(content, "protobuf")
+}
+
+// CommonProtoFileMap returns the map representation of the common protobuf
+// types. This is useful for protoreflect parsing.
+func CommonProtoFileMap() (map[string]string, error) {
+	protoFS, err := CommonProtoFiles()
+	if err != nil {
+		return nil, err
+	}
+
+	once.Do(func() {
+		protoMap = make(map[string]string)
+		err = fs.WalkDir(protoFS, ".", func(path string, d fs.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+			if d.IsDir() || filepath.Ext(path) != ".proto" {
+				return nil
+			}
+			data, err := fs.ReadFile(protoFS, path)
+			if err == nil {
+				protoMap[path] = string(data)
+			}
+			return nil
+		})
+	})
+
+	if err != nil {
+		return nil, err
+	}
+	return protoMap, err
+}

src/go/rpk/pkg/serde/embed/embed_test.go

@@ -0,0 +1,42 @@
+package embed
+
+import (
+	"io/fs"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestEmbeddedFiles(t *testing.T) {
+	t.Run("Test Embedded files in rpk, equal to Redpanda", func(t *testing.T) {
+		// /src/v/pandaproxy/schema_registry/protobuf
+		redpandaProtoFS := os.DirFS("../../../../../v/pandaproxy/schema_registry/protobuf/")
+		redpandaMap := make(map[string]string)
+		err := fs.WalkDir(redpandaProtoFS, ".", func(path string, d fs.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+			if d.IsDir() || filepath.Ext(path) != ".proto" {
+				return nil
+			}
+			data, err := fs.ReadFile(redpandaProtoFS, path)
+			if err == nil {
+				redpandaMap[path] = string(data)
+			}
+			return nil
+		})
+
+		embeddedMap, err := CommonProtoFileMap()
+		require.NoError(t, err)
+
+		for path, embedContent := range embeddedMap {
+			if rpContent, ok := redpandaMap[path]; ok {
+				require.Equalf(t, rpContent, embedContent, "Contents of %v have changed vs the embedded rpk files", path)
+			} else {
+				t.Fatalf("%s not found in Redpanda files", path)
+			}
+		}
+	})
+}

src/go/rpk/pkg/serde/embed/protobuf/.clang-format

@@ -0,0 +1,2 @@
+Language: Proto
+DisableFormat: true

src/go/rpk/pkg/serde/embed/protobuf/confluent/meta.proto

@@ -0,0 +1,27 @@
+syntax = "proto3";
+package confluent;
+
+import "google/protobuf/descriptor.proto";
+
+option go_package="../confluent";
+
+message Meta {
+  string doc = 1;
+  map<string, string> params = 2;
+}
+
+extend google.protobuf.FileOptions {
+  Meta file_meta = 1088;
+}
+extend google.protobuf.MessageOptions {
+  Meta message_meta = 1088;
+}
+extend google.protobuf.FieldOptions {
+  Meta field_meta = 1088;
+}
+extend google.protobuf.EnumOptions {
+  Meta enum_meta = 1088;
+}
+extend google.protobuf.EnumValueOptions {
+  Meta enum_value_meta = 1088;
+}

src/go/rpk/pkg/serde/embed/protobuf/confluent/types/decimal.proto

@@ -0,0 +1,17 @@
+syntax = "proto3";
+
+package confluent.type;
+
+option go_package="../types";
+
+message Decimal {
+
+  // The two's-complement representation of the unscaled integer value in big-endian byte order
+  bytes value = 1;
+
+  // The precision
+  uint32 precision = 2;
+
+  // The scale
+  int32 scale = 3;
+}

src/go/rpk/pkg/serde/embed/protobuf/google/type/README.md

@@ -0,0 +1,16 @@
+# Google Common Types
+
+This package contains definitions of common types for Google APIs.
+All types defined in this package are suitable for different APIs to
+exchange data, and will never break binary compatibility. They should
+have design quality comparable to major programming languages like
+Java and C#.
+
+NOTE: Some common types are defined in the package `google.protobuf`
+as they are directly supported by Protocol Buffers compiler and
+runtime. Those types are called Well-Known Types.
+
+## Java Utilities
+
+A set of Java utilities for the Common Types are provided in the
+`//java/com/google/type/util/` package.

src/go/rpk/pkg/serde/embed/protobuf/google/type/calendar_period.proto

@@ -0,0 +1,56 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package google.type;
+
+option go_package = "google.golang.org/genproto/googleapis/type/calendarperiod;calendarperiod";
+option java_multiple_files = true;
+option java_outer_classname = "CalendarPeriodProto";
+option java_package = "com.google.type";
+option objc_class_prefix = "GTP";
+
+// A `CalendarPeriod` represents the abstract concept of a time period that has
+// a canonical start. Grammatically, "the start of the current
+// `CalendarPeriod`." All calendar times begin at midnight UTC.
+enum CalendarPeriod {
+  // Undefined period, raises an error.
+  CALENDAR_PERIOD_UNSPECIFIED = 0;
+
+  // A day.
+  DAY = 1;
+
+  // A week. Weeks begin on Monday, following
+  // [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date).
+  WEEK = 2;
+
+  // A fortnight. The first calendar fortnight of the year begins at the start
+  // of week 1 according to
+  // [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date).
+  FORTNIGHT = 3;
+
+  // A month.
+  MONTH = 4;
+
+  // A quarter. Quarters start on dates 1-Jan, 1-Apr, 1-Jul, and 1-Oct of each
+  // year.
+  QUARTER = 5;
+
+  // A half-year. Half-years start on dates 1-Jan and 1-Jul.
+  HALF = 6;
+
+  // A year.
+  YEAR = 7;
+}