Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add support for materialized_view #139

Merged
merged 22 commits into from
Mar 18, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ module "bigquery" {
env = "devops"
billable = "true"
owner = "joedoe"
},
}
}
],

Expand Down Expand Up @@ -194,10 +194,11 @@ This module provisions a dataset and a list of tables with associated JSON schem
| encryption\_key | Default encryption key to apply to the dataset. Defaults to null (Google-managed). | `string` | `null` | no |
| external\_tables | A list of objects which include table\_id, expiration\_time, external\_data\_configuration, and labels. | <pre>list(object({<br> table_id = string,<br> autodetect = bool,<br> compression = string,<br> ignore_unknown_values = bool,<br> max_bad_records = number,<br> schema = string,<br> source_format = string,<br> source_uris = list(string),<br> csv_options = object({<br> quote = string,<br> allow_jagged_rows = bool,<br> allow_quoted_newlines = bool,<br> encoding = string,<br> field_delimiter = string,<br> skip_leading_rows = number,<br> }),<br> google_sheets_options = object({<br> range = string,<br> skip_leading_rows = number,<br> }),<br> hive_partitioning_options = object({<br> mode = string,<br> source_uri_prefix = string,<br> }),<br> expiration_time = string,<br> labels = map(string),<br> }))</pre> | `[]` | no |
| location | The regional location for the dataset only US and EU are allowed in module | `string` | `"US"` | no |
| materialized\_views | A list of objects which includes view\_id, view\_query, clustering, time\_partitioning, range\_partitioning, expiration\_time and labels | <pre>list(object({<br> view_id = string,<br> query = string,<br> enable_refresh = bool,<br> refresh_interval_ms = string,<br> clustering = list(string),<br> time_partitioning = object({<br> expiration_ms = string,<br> field = string,<br> type = string,<br> require_partition_filter = bool,<br> }),<br> range_partitioning = object({<br> field = string,<br> range = object({<br> start = string,<br> end = string,<br> interval = string,<br> }),<br> }),<br> expiration_time = string,<br> labels = map(string),<br> }))</pre> | `[]` | no |
| project\_id | Project where the dataset and table are created | `string` | n/a | yes |
| routines | A list of objects which include routine\_id, routine\_type, routine\_language, definition\_body, return\_type, routine\_description and arguments. | <pre>list(object({<br> routine_id = string,<br> routine_type = string,<br> language = string,<br> definition_body = string,<br> return_type = string,<br> description = string,<br> arguments = list(object({<br> name = string,<br> data_type = string,<br> argument_kind = string,<br> mode = string,<br> })),<br> }))</pre> | `[]` | no |
| tables | A list of objects which include table\_id, schema, clustering, time\_partitioning, range\_partitioning, expiration\_time and labels. | <pre>list(object({<br> table_id = string,<br> schema = string,<br> clustering = list(string),<br> time_partitioning = object({<br> expiration_ms = string,<br> field = string,<br> type = string,<br> require_partition_filter = bool,<br> }),<br> range_partitioning = object({<br> field = string,<br> range = object({<br> start = string,<br> end = string,<br> interval = string,<br> }),<br> }),<br> expiration_time = string,<br> labels = map(string),<br> }))</pre> | `[]` | no |
| views | A list of objects which include table\_id, which is view id, and view query | <pre>list(object({<br> view_id = string,<br> query = string,<br> use_legacy_sql = bool,<br> labels = map(string),<br> }))</pre> | `[]` | no |
| views | A list of objects which include view\_id and view query | <pre>list(object({<br> view_id = string,<br> query = string,<br> use_legacy_sql = bool,<br> labels = map(string),<br> }))</pre> | `[]` | no |

## Outputs

Expand Down
55 changes: 51 additions & 4 deletions main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,11 @@
*/

locals {
tables = { for table in var.tables : table["table_id"] => table }
views = { for view in var.views : view["view_id"] => view }
external_tables = { for external_table in var.external_tables : external_table["table_id"] => external_table }
routines = { for routine in var.routines : routine["routine_id"] => routine }
tables = { for table in var.tables : table["table_id"] => table }
views = { for view in var.views : view["view_id"] => view }
materialized_views = { for mat_view in var.materialized_views : mat_view["view_id"] => mat_view }
external_tables = { for external_table in var.external_tables : external_table["table_id"] => external_table }
routines = { for routine in var.routines : routine["routine_id"] => routine }

iam_to_primitive = {
"roles/bigquery.dataOwner" : "OWNER"
Expand Down Expand Up @@ -122,6 +123,52 @@ resource "google_bigquery_table" "view" {
}
}

resource "google_bigquery_table" "materialized_view" {
for_each = local.materialized_views
dataset_id = google_bigquery_dataset.main.dataset_id
friendly_name = each.key
table_id = each.key
labels = each.value["labels"]
clustering = each.value["clustering"]
expiration_time = each.value["expiration_time"]
project = var.project_id
deletion_protection = false

dynamic "time_partitioning" {
for_each = each.value["time_partitioning"] != null ? [each.value["time_partitioning"]] : []
content {
type = time_partitioning.value["type"]
expiration_ms = time_partitioning.value["expiration_ms"]
field = time_partitioning.value["field"]
require_partition_filter = time_partitioning.value["require_partition_filter"]
}
}

dynamic "range_partitioning" {
for_each = each.value["range_partitioning"] != null ? [each.value["range_partitioning"]] : []
content {
field = range_partitioning.value["field"]
range {
start = range_partitioning.value["range"].start
end = range_partitioning.value["range"].end
interval = range_partitioning.value["range"].interval
}
}
}

materialized_view {
query = each.value["query"]
enable_refresh = each.value["enable_refresh"]
refresh_interval_ms = each.value["refresh_interval_ms"]
}

lifecycle {
ignore_changes = [
encryption_configuration # managed by google_bigquery_dataset.main.default_encryption_configuration
]
}
}

resource "google_bigquery_table" "external_table" {
for_each = local.external_tables
dataset_id = google_bigquery_dataset.main.dataset_id
Expand Down
30 changes: 29 additions & 1 deletion variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ variable "tables" {
}

variable "views" {
description = "A list of objects which include table_id, which is view id, and view query"
description = "A list of objects which include view_id and view query"
default = []
type = list(object({
view_id = string,
Expand All @@ -125,6 +125,34 @@ variable "views" {
}))
}

variable "materialized_views" {
description = "A list of objects which includes view_id, view_query, clustering, time_partitioning, range_partitioning, expiration_time and labels"
default = []
type = list(object({
morgante marked this conversation as resolved.
Show resolved Hide resolved
view_id = string,
query = string,
enable_refresh = bool,
refresh_interval_ms = string,
clustering = list(string),
time_partitioning = object({
expiration_ms = string,
field = string,
type = string,
require_partition_filter = bool,
}),
range_partitioning = object({
field = string,
range = object({
start = string,
end = string,
interval = string,
}),
}),
expiration_time = string,
labels = map(string),
}))
}

variable "external_tables" {
description = "A list of objects which include table_id, expiration_time, external_data_configuration, and labels."
default = []
Expand Down