-
Notifications
You must be signed in to change notification settings - Fork 4.1k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Minor fixes to incremental normalization and nesting #7669
Changes from all commits
5353994
b2b3f57
90fbf86
05f28a9
f332c28
b4edde0
bb84c84
40a0f0f
5bfbdb2
62c2992
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,20 @@ | ||
{{ config( | ||
indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], | ||
unique_key = '_airbyte_ab_id', | ||
schema = "test_normalization", | ||
tags = [ "nested" ] | ||
) }} | ||
-- Final base SQL model | ||
select | ||
_airbyte_nested_stre__nto_long_names_hashid, | ||
double_array_data, | ||
{{ adapter.quote('DATA') }}, | ||
_airbyte_ab_id, | ||
_airbyte_emitted_at, | ||
{{ current_timestamp() }} as _airbyte_normalized_at, | ||
_airbyte_partition_hashid | ||
from {{ ref('nested_stream_with_c___long_names_partition_ab3') }} | ||
-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from {{ ref('nested_stream_with_c__lting_into_long_names_scd') }} | ||
where 1 = 1 | ||
{{ incremental_clause('_airbyte_emitted_at') }} | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
{{ config( | ||
indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], | ||
schema = "test_normalization", | ||
tags = [ "nested" ] | ||
) }} | ||
Comment on lines
+1
to
+5
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Un-nesting an array type column from the parent stream can NOT re-use the same (this would fail with exceptions on certain destinations) |
||
-- Final base SQL model | ||
select | ||
_airbyte_partition_hashid, | ||
currency, | ||
_airbyte_ab_id, | ||
_airbyte_emitted_at, | ||
{{ current_timestamp() }} as _airbyte_normalized_at, | ||
_airbyte_data_hashid | ||
from {{ ref('nested_stream_with_c___names_partition_data_ab3') }} | ||
-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from {{ ref('nested_stream_with_c___long_names_partition') }} | ||
where 1 = 1 | ||
{{ incremental_clause('_airbyte_emitted_at') }} | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,7 +1,7 @@ | ||
|
||
|
||
|
||
create table "postgres".test_normalization."dedup_exchange_rate_scd" | ||
create table "postgres"."test_normalization"."dedup_exchange_rate_scd__dbt_tmp" | ||
as ( | ||
|
||
with | ||
|
@@ -42,15 +42,15 @@ scd_data as ( | |
"date" desc, | ||
_airbyte_emitted_at desc | ||
) as _airbyte_end_at, | ||
case when lag("date") over ( | ||
case when row_number() over ( | ||
partition by "id", currency, cast(nzd as | ||
varchar | ||
) | ||
order by | ||
"date" is null asc, | ||
"date" desc, | ||
_airbyte_emitted_at desc | ||
) is null then 1 else 0 end as _airbyte_active_row, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. when all (multiple) rows for the same primary key have a cursor (date here) equal to NULL, then this will flag multiple rows as active... |
||
) = 1 then 1 else 0 end as _airbyte_active_row, | ||
_airbyte_ab_id, | ||
_airbyte_emitted_at, | ||
_airbyte_dedup_exchange_rate_hashid | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -31,6 +31,9 @@ | |
}, | ||
"USD": { | ||
"type": "number" | ||
}, | ||
"column`_'with\"_quotes": { | ||
"type": "string" | ||
} | ||
} | ||
}, | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -31,6 +31,9 @@ | |
}, | ||
"USD": { | ||
"type": "number" | ||
}, | ||
"column`_'with\"_quotes": { | ||
"type": "string" | ||
} | ||
} | ||
}, | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Un-nesting an object type column from the parent stream can re-use the same
_airbyte_ab_id
column as unique_key