Add sinks to write to GCP Bigquery

.github/actions/spelling/allow.txt

@@ -244,6 +244,7 @@ armhf
 backpressure
 backticks
 bigendian
+bigquery
 bindir
 binfmt
 bitcast

.github/actions/spelling/excludes.txt

@@ -118,4 +118,5 @@
 ^\Qwebsite/layouts/shortcodes/config/unit-tests.html\E$
 ^lib/codecs/tests/data/native_encoding/
 ^\Qwebsite/config.toml\E$
+^proto/google/
 ignore$

.github/actions/spelling/expect.txt

@@ -524,6 +524,7 @@ ingesters
 ingestor
 initdb
 initech
+Insertdata
 installdeb
 Instrumentable
 interpolatedstring

buf.yaml

@@ -3,6 +3,7 @@ modules:
   - path: lib/vector-core/proto
   - path: proto/third-party
   - path: proto/vector
+  - path: proto/google/cloud/bigquery/storage/v1
 lint:
   use:
     - DEFAULT

build.rs

@@ -123,6 +123,7 @@
         println!("cargo:rerun-if-changed=proto/vector/dd_trace.proto");
         println!("cargo:rerun-if-changed=proto/vector/ddsketch_full.proto");
         println!("cargo:rerun-if-changed=proto/vector/vector.proto");
+        println!("cargo:rerun-if-changed=proto/google/cloud/bigquery/storage/v1/storage.proto");
 
         // Create and store the "file descriptor set" from the compiled Protocol Buffers packages.
         //
@@ -144,6 +145,7 @@
                 prost_build,
                 &[
                     "lib/vector-core/proto/event.proto",
+                    "proto/google/cloud/bigquery/storage/v1/storage.proto",
                     "proto/third-party/dnstap.proto",
                     "proto/vector/ddsketch_full.proto",
                     "proto/vector/dd_metric.proto",
@@ -155,6 +157,7 @@
                 &[
                     "proto/third-party",
                     "proto/vector",
+                    "proto/gogole/cloud/bigquery/storage/v1",
                     "lib/vector-core/proto/",
                 ],
             )
@@ -268,3 +271,4 @@
     // Emit the aforementioned stanzas.
     tracker.emit_rerun_stanzas();
 }
+

lib/vector-core/build.rs

@@ -6,7 +6,7 @@ fn main() {
         .bytes(["raw_bytes"])
         .compile_protos(
             &["proto/event.proto"],
-            &["proto", "../../proto/third-party", "../../proto/vector"],
+            &["proto", "../../proto/third-party", "../../proto/vector",  "../../proto/google/cloud/bigquery/storage/v1"],
         )
         .unwrap();
 }

proto/google/cloud/bigquery/storage/v1/arrow.proto

@@ -0,0 +1,64 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package google.cloud.bigquery.storage.v1;
+
+option csharp_namespace = "Google.Cloud.BigQuery.Storage.V1";
+option go_package = "cloud.google.com/go/bigquery/storage/apiv1/storagepb;storagepb";
+option java_multiple_files = true;
+option java_outer_classname = "ArrowProto";
+option java_package = "com.google.cloud.bigquery.storage.v1";
+option php_namespace = "Google\\Cloud\\BigQuery\\Storage\\V1";
+
+// Arrow schema as specified in
+// https://arrow.apache.org/docs/python/api/datatypes.html
+// and serialized to bytes using IPC:
+// https://arrow.apache.org/docs/format/Columnar.html#serialization-and-interprocess-communication-ipc
+//
+// See code samples on how this message can be deserialized.
+message ArrowSchema {
+  // IPC serialized Arrow schema.
+  bytes serialized_schema = 1;
+}
+
+// Arrow RecordBatch.
+message ArrowRecordBatch {
+  // IPC-serialized Arrow RecordBatch.
+  bytes serialized_record_batch = 1;
+
+  // [Deprecated] The count of rows in `serialized_record_batch`.
+  // Please use the format-independent ReadRowsResponse.row_count instead.
+  int64 row_count = 2 [deprecated = true];
+}
+
+// Contains options specific to Arrow Serialization.
+message ArrowSerializationOptions {
+  // Compression codec's supported by Arrow.
+  enum CompressionCodec {
+    // If unspecified no compression will be used.
+    COMPRESSION_UNSPECIFIED = 0;
+
+    // LZ4 Frame (https://github.com/lz4/lz4/blob/dev/doc/lz4_Frame_format.md)
+    LZ4_FRAME = 1;
+
+    // Zstandard compression.
+    ZSTD = 2;
+  }
+
+  // The compression codec to use for Arrow buffers in serialized record
+  // batches.
+  CompressionCodec buffer_compression = 2;
+}

proto/google/cloud/bigquery/storage/v1/avro.proto

@@ -0,0 +1,56 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package google.cloud.bigquery.storage.v1;
+
+option csharp_namespace = "Google.Cloud.BigQuery.Storage.V1";
+option go_package = "cloud.google.com/go/bigquery/storage/apiv1/storagepb;storagepb";
+option java_multiple_files = true;
+option java_outer_classname = "AvroProto";
+option java_package = "com.google.cloud.bigquery.storage.v1";
+option php_namespace = "Google\\Cloud\\BigQuery\\Storage\\V1";
+
+// Avro schema.
+message AvroSchema {
+  // Json serialized schema, as described at
+  // https://avro.apache.org/docs/1.8.1/spec.html.
+  string schema = 1;
+}
+
+// Avro rows.
+message AvroRows {
+  // Binary serialized rows in a block.
+  bytes serialized_binary_rows = 1;
+
+  // [Deprecated] The count of rows in the returning block.
+  // Please use the format-independent ReadRowsResponse.row_count instead.
+  int64 row_count = 2 [deprecated = true];
+}
+
+// Contains options specific to Avro Serialization.
+message AvroSerializationOptions {
+  // Enable displayName attribute in Avro schema.
+  //
+  // The Avro specification requires field names to be alphanumeric.  By
+  // default, in cases when column names do not conform to these requirements
+  // (e.g. non-ascii unicode codepoints) and Avro is requested as an output
+  // format, the CreateReadSession call will fail.
+  //
+  // Setting this field to true, populates avro field names with a placeholder
+  // value and populates a "displayName" attribute for every avro field with the
+  // original column name.
+  bool enable_display_name_attribute = 1;
+}

proto/google/cloud/bigquery/storage/v1/protobuf.proto

@@ -0,0 +1,48 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package google.cloud.bigquery.storage.v1;
+
+import "google/protobuf/descriptor.proto";
+
+option csharp_namespace = "Google.Cloud.BigQuery.Storage.V1";
+option go_package = "cloud.google.com/go/bigquery/storage/apiv1/storagepb;storagepb";
+option java_multiple_files = true;
+option java_outer_classname = "ProtoBufProto";
+option java_package = "com.google.cloud.bigquery.storage.v1";
+option php_namespace = "Google\\Cloud\\BigQuery\\Storage\\V1";
+
+// ProtoSchema describes the schema of the serialized protocol buffer data rows.
+message ProtoSchema {
+  // Descriptor for input message.  The provided descriptor must be self
+  // contained, such that data rows sent can be fully decoded using only the
+  // single descriptor.  For data rows that are compositions of multiple
+  // independent messages, this means the descriptor may need to be transformed
+  // to only use nested types:
+  // https://developers.google.com/protocol-buffers/docs/proto#nested
+  //
+  // For additional information for how proto types and values map onto BigQuery
+  // see: https://cloud.google.com/bigquery/docs/write-api#data_type_conversions
+  google.protobuf.DescriptorProto proto_descriptor = 1;
+}
+
+message ProtoRows {
+  // A sequence of rows serialized as a Protocol Buffer.
+  //
+  // See https://developers.google.com/protocol-buffers/docs/overview for more
+  // information on deserializing this field.
+  repeated bytes serialized_rows = 1;
+}