Skip to content

Commit

Permalink
Removed support for external table data from Bittable (beta) from thi…
Browse files Browse the repository at this point in the history 
…s PR
  • Loading branch information
@rickard-von-essen @modular-magician
rickard-von-essen authored and modular-magician committed Jul 3, 2019
1 parent 5c883ce commit 6165666
Show file tree
Hide file tree
Showing 2 changed files with 2 additions and 143 deletions.
124 changes: 1 addition & 123 deletions third_party/terraform/resources/resource_bigquery_table.go.erb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,44 +80,6 @@ func resourceBigQueryTable() *schema.Resource {
Type: schema.TypeBool,
Required: true,
},
// BigtableOptions: [Optional] Additional options if sourceFormat is set to BIGTABLE.
"bigtable_options": {
<% if version.nil? || version == 'ga' -%>
Removed: "This field is in beta. Use it in the the google-beta provider instead. See https://terraform.io/docs/providers/google/provider_versions.html for more details.",
<% end -%>
Type: schema.TypeList,
Optional: true,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
// ColumnFamilies: [Optional] List of column families to expose in the
// table schema along with their types.
"column_families": {
Type: schema.TypeString,
Optional: true,
Computed: true,
ValidateFunc: validation.ValidateJsonString,
StateFunc: func(v interface{}) string {
json, _ := structure.NormalizeJsonString(v)
return json
},
},
// IgnoreUnspecifiedColumnFamilies: [Optional] If field is true, then
// the column families that are not specified in columnFamilies list are
// not exposed in the table schema.
"ignore_unspecified_column_families": {
Type: schema.TypeBool,
Optional: true,
},
// ReadRowkeyAsString: [Optional] If field is true, then the rowkey
// column families will be read and converted to string.
"read_rowkey_as_string": {
Type: schema.TypeBool,
Optional: true,
},
},
},
},
// Compression: [Optional] The compression type of the data source.
"compression": {
Type: schema.TypeString,
Expand Down Expand Up @@ -233,7 +195,7 @@ func resourceBigQueryTable() *schema.Resource {
Type: schema.TypeString,
Required: true,
ValidateFunc: validation.StringInSlice([]string{
"CSV", "GOOGLE_SHEETS", "NEWLINE_DELIMITED_JSON", "AVRO", "DATSTORE_BACKUP", <%= version != 'ga' ? ' "BIGTABLE",' : '' %>
"CSV", "GOOGLE_SHEETS", "NEWLINE_DELIMITED_JSON", "AVRO", "DATSTORE_BACKUP",
}, false),
},
// SourceURIs [Required] The fully-qualified URIs that point to your data in Google Cloud.
Expand Down Expand Up @@ -630,13 +592,6 @@ func expandExternalDataConfiguration(cfg interface{}) (*bigquery.ExternalDataCon
edc.SourceUris = sourceUris
}

if v, ok := raw["bigtable_options"]; ok {
bigtableOpts, err := expandBigtableOptions(v)
if err != nil {
return nil, err
}
edc.BigtableOptions = bigtableOpts
}
if v, ok := raw["compression"]; ok {
edc.Compression = v.(string)
}
Expand Down Expand Up @@ -675,14 +630,6 @@ func flattenExternalDataConfiguration(edc *bigquery.ExternalDataConfiguration) (
result["autodetect"] = edc.Autodetect
result["source_uris"] = edc.SourceUris

if edc.BigtableOptions != nil {
columnFamilies, err := flattenBigtableOptions(edc.BigtableOptions)
if err != nil {
return result, err
}
result["bigtable_options"] = columnFamilies
}

if edc.Compression != "" {
result["compression"] = edc.Compression
}
Expand Down Expand Up @@ -717,56 +664,6 @@ func flattenExternalDataConfiguration(edc *bigquery.ExternalDataConfiguration) (
return result, nil
}

func expandBigtableOptions(configured interface{}) (*bigquery.BigtableOptions, error) {
if len(configured.([]interface{})) == 0 {
return nil, nil
}

raw := configured.([]interface{})[0].(map[string]interface{})
opts := &bigquery.BigtableOptions{}

if v, ok := raw["column_families"]; ok {
columnFamilies, err := expandColumnFamilies(v)
if err != nil {
return nil, err
}

opts.ColumnFamilies = columnFamilies
}

if v, ok := raw["ignore_unspecified_column_families"]; ok {
opts.IgnoreUnspecifiedColumnFamilies = v.(bool)
}

if v, ok := raw["read_rowkey_as_string"]; ok {
opts.ReadRowkeyAsString = v.(bool)
}

return opts, nil
}

func flattenBigtableOptions(opts *bigquery.BigtableOptions) ([]map[string]interface{}, error) {
result := map[string]interface{}{}

if opts.ColumnFamilies != nil {
columnFamilies, err := flattenColumnFamilies(opts.ColumnFamilies)
if err != nil {
return []map[string]interface{}{result}, err
}
result["column_families"] = columnFamilies
}

if opts.IgnoreUnspecifiedColumnFamilies == true {
result["ignore_unspecified_column_families"] = opts.IgnoreUnspecifiedColumnFamilies
}

if opts.ReadRowkeyAsString == true {
result["read_rowkey_as_string"] = opts.ReadRowkeyAsString
}

return []map[string]interface{}{result}, nil
}

func expandCsvOptions(configured interface{}) *bigquery.CsvOptions {
if len(configured.([]interface{})) == 0 {
return nil
Expand Down Expand Up @@ -865,25 +762,6 @@ func flattenGoogleSheetsOptions(opts *bigquery.GoogleSheetsOptions) []map[string
return []map[string]interface{}{result}
}

func expandColumnFamilies(raw interface{}) ([]*bigquery.BigtableColumnFamily, error) {
var columnFamilies []*bigquery.BigtableColumnFamily

if err := json.Unmarshal([]byte(raw.(string)), &columnFamilies); err != nil {
return nil, err
}

return columnFamilies, nil
}

func flattenColumnFamilies(columnFamilies []*bigquery.BigtableColumnFamily) (string, error) {
cf, err := json.Marshal(columnFamilies)
if err != nil {
return "", err
}

return string(cf), nil
}

func expandSchema(raw interface{}) (*bigquery.TableSchema, error) {
var fields []*bigquery.TableFieldSchema

Expand Down
21 changes: 1 addition & 20 deletions third_party/terraform/website/docs/r/bigquery_table.html.markdown
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,10 +120,6 @@ The `external_data_configuration` block supports:
* `autodetect` - (Required) - Let BigQuery try to autodetect the schema
and format of the table.

* `bigtable_options` (Optional, Beta) - Additional options if
`source_format` is set to "BIGTABLE". Structure is
documented below.

* `compression` (Optional) - The compression type of the data source.
Valid values are "NONE" or "GZIP".

Expand Down Expand Up @@ -152,28 +148,13 @@ The `external_data_configuration` block supports:

* `schema_format` (Required) - The data format. Supported values are:
"CVS", "GOOGLE_SHEETS", "NEWLINE_DELIMITED_JSON", "AVRO",
"DATSTORE_BACKUP", and "BIGTABLE" (Beta). To use "GOOGLE_SHEETS"
and "DATSTORE_BACKUP". To use "GOOGLE_SHEETS"
the `scopes` must include
"https://www.googleapis.com/auth/drive.readonly".

* `source_uris` - (Required) A list of the fully-qualified URIs that point to
your data in Google Cloud.

The `bigtable_options` block supports:

* `column_families` (Optional) - A JSON document describing the column families.
For more information see the
[BigQuery API documentation](https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#resource).

* `ignore_unspecified_column_families` (Optional) - If field is true, then
the column families that are not specified in "column_families" list are
not exposed in the table schema. Otherwise, they are read with BYTES
type values. The default value is false.

* `read_rowkey_as_string` (Optional) - If field is true, then the rowkey
column families will be read and converted to string. The default
value is false.

The `cvs_options` block supports:

* `allow_jagged_rows` (Optional) - Indicates if BigQuery should accept rows
Expand Down

0 comments on commit 6165666

Please sign in to comment.