Skip to content

Seamlessly save and load protocol buffers to and from BigQuery using Go.

License

Notifications You must be signed in to change notification settings

einride/protobuf-bigquery-go

Repository files navigation

Protobuf + BigQuery + Go

PkgGoDev GoReportCard Codecov

Seamlessly save and load protocol buffers to and from BigQuery using Go.

This library provides add-ons to cloud.google.com/bigquery for first-class protobuf support using protobuf reflection.

Installing

$ go get -u go.einride.tech/protobuf-bigquery

Examples

protobq.InferSchema

BigQuery schema inference for arbitrary protobuf messages.

func ExampleInferSchema() {
	msg := &library.Book{}
	schema := protobq.InferSchema(msg)
	expected := bigquery.Schema{
		{Name: "name", Type: bigquery.StringFieldType},
		{Name: "author", Type: bigquery.StringFieldType},
		{Name: "title", Type: bigquery.StringFieldType},
		{Name: "read", Type: bigquery.BooleanFieldType},
	}
	fmt.Println(cmp.Equal(expected, schema))
	// Output: true
}

protobq.MessageSaver

An implementation of bigquery.ValueSaver that saves arbitrary protobuf messages to BigQuery.

func ExampleMessageSaver() {
	ctx := context.Background()
	// Write protobuf messages to a BigQuery table.
	projectID := flag.String("project", "", "BigQuery project to write to.")
	datasetID := flag.String("dataset", "", "BigQuery dataset to write to.")
	tableID := flag.String("table", "", "BigQuery table to write to.")
	create := flag.Bool("create", false, "Flag indicating whether to create the table.")
	flag.Parse()
	// Connect to BigQuery.
	client, err := bigquery.NewClient(ctx, *projectID)
	if err != nil {
		panic(err) // TODO: Handle error.
	}
	table := client.Dataset(*datasetID).Table(*tableID)
	// Create the table by inferring the BigQuery schema from the protobuf schema.
	if *create {
		if err := table.Create(ctx, &bigquery.TableMetadata{
			Schema: protobq.InferSchema(&publicv1.FilmLocation{}),
		}); err != nil {
			panic(err) // TODO: Handle error.
		}
	}
	// Insert the protobuf messages.
	inserter := table.Inserter()
	for i, filmLocation := range []*publicv1.FilmLocation{
		{Title: "Dark Passage", ReleaseYear: 1947, Locations: "Filbert Steps"},
		{Title: "D.O.A", ReleaseYear: 1950, Locations: "Union Square"},
		{Title: "Flower Drum Song", ReleaseYear: 1961, Locations: "Chinatown"},
	} {
		if err := inserter.Put(ctx, &protobq.MessageSaver{
			Message:  filmLocation,
			InsertID: strconv.Itoa(i), // include an optional insert ID
		}); err != nil {
			panic(err) // TODO: Handle error.
		}
	}
}

protobq.MessageLoader

An implementation of bigquery.ValueLoader that loads arbitrary protobuf messages from BigQuery.

func ExampleMessageLoader() {
	ctx := context.Background()
	// Read from the public "film locations" BigQuery dataset into a proto message.
	const (
		project = "bigquery-public-data"
		dataset = "san_francisco_film_locations"
		table   = "film_locations"
	)
	// Connect to BigQuery.
	client, err := bigquery.NewClient(ctx, project)
	if err != nil {
		panic(err) // TODO: Handle error.
	}
	// Load BigQuery rows into a FilmLocation message.
	messageLoader := &protobq.MessageLoader{
		Message: &publicv1.FilmLocation{},
	}
	// Iterate rows in table.
	rowIterator := client.Dataset(dataset).Table(table).Read(ctx)
	for {
		// Load next row into the FilmLocation message.
		if err := rowIterator.Next(messageLoader); err != nil {
			if errors.Is(err, iterator.Done) {
				break
			}
			panic(err) // TODO: Handle error.
		}
		// Print the message.
		fmt.Println(prototext.Format(messageLoader.Message))
	}
}

Features

Support for Well-Known Types (google.protobuf)

Protobuf BigQuery
google.protobuf.Timestamp TIMESTAMP
google.protobuf.Duration FLOAT (seconds)
google.protobuf.DoubleValue FLOAT
google.protobuf.FloatValue FLOAT
google.protobuf.Int32Value INTEGER
google.protobuf.Int64Value INTEGER
google.protobuf.Uint32Value INTEGER
google.protobuf.Uint64Value INTEGER
google.protobuf.BoolValue BOOLEAN
google.protobuf.StringValue STRING
google.protobuf.BytesValue BYTES
google.protobuf.StructValue STRING (JSON)

Reference ≫

Support for API Common Protos (google.type)

Protobuf BigQuery
google.type.Date DATE
google.type.DateTime RECORD (or DATETIME)
google.type.LatLng GEOGRAPHY
google.type.TimeOfDay TIME

Reference ≫