Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[CORE-2144] schema_registry/protobuf: Disable protobuf logging #22633

Merged
merged 2 commits into from
Jul 30, 2024
Merged

[CORE-2144] schema_registry/protobuf: Disable protobuf logging #22633

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e4b64db
schema_registry: Translate base64_decoder_exception
BenPope Jul 29, 2024
3aacc02
schema_registry: Disable protobuf logging
BenPope Jul 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions src/v/pandaproxy/schema_registry/protobuf.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,10 +231,14 @@ class parser {

// Attempt parse a .proto file
if (!_parser.Parse(&t, &_fdp)) {
// base64 decode the schema
iobuf_istream is{base64_to_iobuf(schema.def().raw()())};
// Attempt parse as an encoded FileDescriptorProto.pb
if (!_fdp.ParseFromIstream(&is.istream())) {
try {
// base64 decode the schema
iobuf_istream is{base64_to_iobuf(schema.def().raw()())};
// Attempt parse as an encoded FileDescriptorProto.pb
if (!_fdp.ParseFromIstream(&is.istream())) {
throw as_exception(error_collector.error());
}
} catch (const base64_decoder_exception&) {
throw as_exception(error_collector.error());
}
}
Expand Down
35 changes: 35 additions & 0 deletions src/v/pandaproxy/schema_registry/test/compatibility_protobuf.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,41 @@ SEASTAR_THREAD_TEST_CASE(test_protobuf_recursive_reference) {
.get();
}

SEASTAR_THREAD_TEST_CASE(test_binary_protobuf) {
simple_sharded_store store;

BOOST_REQUIRE_NO_THROW(store.store
.make_valid_schema(pps::canonical_schema{
pps::subject{"com.redpanda.Payload.proto"},
pps::canonical_schema_definition{
base64_raw_proto, pps::schema_type::protobuf}})
.get());
}

SEASTAR_THREAD_TEST_CASE(test_invalid_binary_protobuf) {
simple_sharded_store store;

auto broken_base64_raw_proto = base64_raw_proto.substr(1);

auto schema = pps::canonical_schema{
pps::subject{"com.redpanda.Payload.proto"},
pps::canonical_schema_definition{
broken_base64_raw_proto, pps::schema_type::protobuf}};

BOOST_REQUIRE_EXCEPTION(
store.store
.make_valid_schema(pps::canonical_schema{
pps::subject{"com.redpanda.Payload.proto"},
pps::canonical_schema_definition{
broken_base64_raw_proto, pps::schema_type::protobuf}})
.get(),
pps::exception,
[](const pps::exception& e) {
std::cout << e.what();
return e.code() == pps::error_code::schema_invalid;
});
}

SEASTAR_THREAD_TEST_CASE(test_protobuf_well_known) {
simple_sharded_store store;

Expand Down
67 changes: 67 additions & 0 deletions src/v/pandaproxy/schema_registry/test/compatibility_protobuf.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,3 +88,70 @@ message A1 {
}
})",
pps::schema_type::protobuf};

// Binary encoded protobuf descriptor:
//
// syntax = "proto3";
//
// package com.redpanda;
//
// option go_package = "./;main";
// option java_multiple_files = true;
//
// import "google/protobuf/timestamp.proto";
//
// message Payload {
// int32 val = 1;
// google.protobuf.Timestamp timestamp = 2;
// }
//
// message A {
// message B {
// message C {
// message D {
// message M00 {}
// message M01 {}
// message M02 {}
// message M03 {}
// message M04 {}
// message M05 {}
// message M06 {}
// message M07 {}
// message M08 {}
// message M09 {}
// message M10 {}
// message M11 {}
// message M12 {}
// message M13 {}
// message M14 {}
// message M15 {}
// message M16 {}
// message M17 {}
// message NestedPayload {
// int32 val = 1;
// google.protobuf.Timestamp timestamp = 2;
// }
// }
// }
// }
// }
//
// message CompressiblePayload {
// int32 val = 1;
// google.protobuf.Timestamp timestamp = 2;
// string message = 3;
// }
constexpr std::string_view base64_raw_proto{
"Cg1wYXlsb2FkLnByb3RvEgxjb20ucmVkcGFuZGEaH2dvb2dsZS9wcm90b2J1Zi90aW1lc3RhbXAu"
"cHJvdG8iRQoHUGF5bG9hZBILCgN2YWwYASABKAUSLQoJdGltZXN0YW1wGAIgASgLMhouZ29vZ2xl"
"LnByb3RvYnVmLlRpbWVzdGFtcCLgAQoBQRraAQoBQhrUAQoBQxrOAQoBRBoFCgNNMDAaBQoDTTAx"
"GgUKA00wMhoFCgNNMDMaBQoDTTA0GgUKA00wNRoFCgNNMDYaBQoDTTA3GgUKA00wOBoFCgNNMDka"
"BQoDTTEwGgUKA00xMRoFCgNNMTIaBQoDTTEzGgUKA00xNBoFCgNNMTUaBQoDTTE2GgUKA00xNxpL"
"Cg1OZXN0ZWRQYXlsb2FkEgsKA3ZhbBgBIAEoBRItCgl0aW1lc3RhbXAYAiABKAsyGi5nb29nbGUu"
"cHJvdG9idWYuVGltZXN0YW1wImIKE0NvbXByZXNzaWJsZVBheWxvYWQSCwoDdmFsGAEgASgFEi0K"
"CXRpbWVzdGFtcBgCIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASDwoHbWVzc2FnZRgD"
"IAEoCUILUAFaBy4vO21haW5iBnByb3RvMw=="};

const pps::canonical_schema_definition base64_proto{
pps::canonical_schema_definition{
base64_raw_proto, pps::schema_type::protobuf}};
7 changes: 7 additions & 0 deletions src/v/redpanda/application.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,7 @@
#include <seastar/util/defer.hh>
#include <seastar/util/log.hh>

#include <google/protobuf/stubs/logging.h>
#include <sys/resource.h>
#include <sys/utsname.h>

Expand Down Expand Up @@ -525,6 +526,12 @@ void application::initialize(
.get();
_cpu_profiler.invoke_on_all(&resources::cpu_profiler::start).get();

/*
* Disable the logger for protobuf; some interfaces don't allow a pluggable
* error collector.
*/
google::protobuf::SetLogHandler(nullptr);

Comment on lines +529 to +534
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess this is fine... is there any information we'll lose by setting this to nullptr?

Copy link
Member Author

@BenPope BenPope Jul 30, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess this is fine... is there any information we'll lose by setting this to nullptr?

Ideally the interface would take an error collector, such as:

But in this case, not. The only interface that I'm using that doesn't have a collector is the one that produces the this spurious log, which is nearly always wrong, since if it can't decode as plain text, it attempts to decode as proto encoded proto, which is highly likely to succeed. In any case, we are not interested in the information in the log.

I added @jcipar as they also have been using protobuf.

/*
* allocate per-core zstd decompression workspace and per-core
* async_stream_zstd workspaces. it can be several megabytes in size, so do
Expand Down
Loading