From 47da43b14101aa93eba0a9c80a9c8d73a14a6048 Mon Sep 17 00:00:00 2001 From: vlad-ogol <103129145+vlad-ogol@users.noreply.github.com> Date: Thu, 1 Feb 2024 11:40:32 +0200 Subject: [PATCH] Allow 'partitionBy' and 'clusterBy' params for bigquery materialized views (#1660) * Allow 'partitionBy' and 'clusterBy' params for bigquery materialized views * Bump version to 2.9.0 --- core/session.ts | 16 ++++++++++++++-- tests/core/core.spec.ts | 42 ++++++++++++++++++++++++++++++++++++++--- version.bzl | 2 +- 3 files changed, 54 insertions(+), 6 deletions(-) diff --git a/core/session.ts b/core/session.ts index adc40096e..25ac34b58 100644 --- a/core/session.ts +++ b/core/session.ts @@ -730,10 +730,22 @@ export class Session { table.bigquery.clusterBy?.length || table.bigquery.partitionExpirationDays || table.bigquery.requirePartitionFilter) && - table.enumType === dataform.TableType.VIEW + table.enumType === dataform.TableType.VIEW && + !table.materialized ) { this.compileError( - `partitionBy/clusterBy/requirePartitionFilter/partitionExpirationDays are not valid for BigQuery views; they are only valid for tables`, + `partitionBy/clusterBy/requirePartitionFilter/partitionExpirationDays are not valid for BigQuery views`, + table.fileName, + table.target + ); + } else if ( + (table.bigquery.partitionExpirationDays || + table.bigquery.requirePartitionFilter) && + table.enumType === dataform.TableType.VIEW && + table.materialized + ) { + this.compileError( + `requirePartitionFilter/partitionExpirationDays are not valid for BigQuery materialized views`, table.fileName, table.target ); diff --git a/tests/core/core.spec.ts b/tests/core/core.spec.ts index 476cd2901..a76a3d494 100644 --- a/tests/core/core.spec.ts +++ b/tests/core/core.spec.ts @@ -513,6 +513,30 @@ suite("@dataform/core", () => { partitionExpirationDays: 7 } }); + session.publish("example_require_partition_filter_view_fail", { + type: "view", + bigquery: { + requirePartitionFilter: true + } + }); + session.publish("example_expiring_materialized_view_fail", { + type: "view", + materialized: true, + bigquery: { + partitionBy: "some_partition", + clusterBy: ["some_cluster"], + partitionExpirationDays: 7 + } + }); + session.publish("example_require_partition_filter_materialized_view_fail", { + type: "view", + materialized: true, + bigquery: { + partitionBy: "some_partition", + clusterBy: ["some_cluster"], + requirePartitionFilter: true + } + }); session.publish("example_materialize_table_fail", { type: "table", materialized: true @@ -554,15 +578,27 @@ suite("@dataform/core", () => { ).has.deep.members([ { actionName: "schema.example_partitionBy_view_fail", - message: `partitionBy/clusterBy/requirePartitionFilter/partitionExpirationDays are not valid for BigQuery views; they are only valid for tables` + message: `partitionBy/clusterBy/requirePartitionFilter/partitionExpirationDays are not valid for BigQuery views` }, { actionName: "schema.example_clusterBy_view_fail", - message: `partitionBy/clusterBy/requirePartitionFilter/partitionExpirationDays are not valid for BigQuery views; they are only valid for tables` + message: `partitionBy/clusterBy/requirePartitionFilter/partitionExpirationDays are not valid for BigQuery views` }, { actionName: "schema.example_expiring_view_fail", - message: `partitionBy/clusterBy/requirePartitionFilter/partitionExpirationDays are not valid for BigQuery views; they are only valid for tables` + message: `partitionBy/clusterBy/requirePartitionFilter/partitionExpirationDays are not valid for BigQuery views` + }, + { + actionName: "schema.example_require_partition_filter_view_fail", + message: `partitionBy/clusterBy/requirePartitionFilter/partitionExpirationDays are not valid for BigQuery views` + }, + { + actionName: "schema.example_expiring_materialized_view_fail", + message: `requirePartitionFilter/partitionExpirationDays are not valid for BigQuery materialized views` + }, + { + actionName: "schema.example_require_partition_filter_materialized_view_fail", + message: `requirePartitionFilter/partitionExpirationDays are not valid for BigQuery materialized views` }, { actionName: "schema.example_materialize_table_fail", diff --git a/version.bzl b/version.bzl index cd026e13a..dc1a090e6 100644 --- a/version.bzl +++ b/version.bzl @@ -1,3 +1,3 @@ # NOTE: If you change the format of this line, you must change the bash command # in /scripts/publish to extract the version string correctly. -DF_VERSION = "2.8.4" +DF_VERSION = "2.9.0"