Skip to content

Commit

Permalink
add test for static overwrite day case
Browse files Browse the repository at this point in the history
  • Loading branch information
Kayrnt committed Jul 13, 2023
1 parent d62cfe7 commit 124023f
Show file tree
Hide file tree
Showing 2 changed files with 52 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -450,3 +450,50 @@
where date_time > '2020-01-01'
{% endif %}
""".lstrip()

overwrite_static_day_sql = """
{% set partitions_to_replace = [
"'2020-01-01'",
"'2020-01-02'",
] %}
{{
config(
materialized="incremental",
incremental_strategy="insert_overwrite",
cluster_by="id",
partition_by={
"field": "date_time",
"data_type": "datetime",
"granularity": "day"
},
partitions=partitions_to_replace,
on_schema_change="sync_all_columns"
)
}}
with data as (
{% if not is_incremental() %}
select 1 as id, cast('2020-01-01' as datetime) as date_time union all
select 2 as id, cast('2020-01-01' as datetime) as date_time union all
select 3 as id, cast('2020-01-01' as datetime) as date_time union all
select 4 as id, cast('2020-01-01' as datetime) as date_time
{% else %}
-- we want to overwrite the 4 records in the 2020-01-01 partition
-- with the 2 records below, but add two more in the 2020-01-02 partition
select 10 as id, cast('2020-01-01' as datetime) as date_time union all
select 20 as id, cast('2020-01-01' as datetime) as date_time union all
select 30 as id, cast('2020-01-02' as datetime) as date_time union all
select 40 as id, cast('2020-01-02' as datetime) as date_time
{% endif %}
)
select * from data
""".lstrip()
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
overwrite_time_sql,
overwrite_day_with_time_ingestion_sql,
overwrite_day_with_time_partition_datetime_sql,
overwrite_static_day_sql,
)


Expand All @@ -46,6 +47,7 @@ def models(self):
"incremental_overwrite_time.sql": overwrite_time_sql,
"incremental_overwrite_day_with_time_partition.sql": overwrite_day_with_time_ingestion_sql,
"incremental_overwrite_day_with_time_partition_datetime.sql": overwrite_day_with_time_partition_datetime_sql,
"incremental_overwrite_static_day.sql": overwrite_static_day_sql,
}

@pytest.fixture(scope="class")
Expand All @@ -63,10 +65,10 @@ def seeds(self):
def test__bigquery_assert_incremental_configurations_apply_the_right_strategy(self, project):
run_dbt(["seed"])
results = run_dbt()
assert len(results) == 10
assert len(results) == 11

results = run_dbt()
assert len(results) == 10
assert len(results) == 11
incremental_strategies = [
("incremental_merge_range", "merge_expected"),
("incremental_merge_time", "merge_expected"),
Expand All @@ -79,6 +81,7 @@ def test__bigquery_assert_incremental_configurations_apply_the_right_strategy(se
"incremental_overwrite_day_with_time_partition_datetime",
"incremental_overwrite_day_with_time_partition_expected",
),
("incremental_overwrite_static_day", "incremental_overwrite_day_expected"),
]
db_with_schema = f"{project.database}.{project.test_schema}"
for incremental_strategy in incremental_strategies:
Expand Down

0 comments on commit 124023f

Please sign in to comment.