diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql deleted file mode 100644 index dae3ef07ec98..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ /dev/null @@ -1,13 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_array(_airbyte_data, "$['conflict_stream_array']") as conflict_stream_array, - _airbyte_emitted_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_array as table_alias --- conflict_stream_array; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql deleted file mode 100644 index 43a66e1de59b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - string -) as id, - conflict_stream_array, - _airbyte_emitted_at -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab1` --- conflict_stream_array; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql deleted file mode 100644 index 4854b7bdf8e2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(array_to_string(conflict_stream_array, "|", "") as - string -), '')) as - string -))) as _airbyte_conflict_stream_array_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab2` tmp --- conflict_stream_array; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql deleted file mode 100644 index b2a5e728a659..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - - json_extract(table_alias._airbyte_data, "$['conflict_stream_name']") - as conflict_stream_name, - _airbyte_emitted_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_name as table_alias --- conflict_stream_name; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql deleted file mode 100644 index 0e880d8d1132..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - string -) as id, - cast(conflict_stream_name as - string -) as conflict_stream_name, - _airbyte_emitted_at -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab1` --- conflict_stream_name; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql deleted file mode 100644 index b1c04efa9773..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(conflict_stream_name as - string -), '')) as - string -))) as _airbyte_conflict_stream_name_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab2` tmp --- conflict_stream_name; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql deleted file mode 100644 index 15ff7f7cc272..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_hashid, - - json_extract(table_alias.conflict_stream_name, "$['conflict_stream_name']") - as conflict_stream_name, - _airbyte_emitted_at -from `dataline-integration-testing`.test_normalization.`conflict_stream_name` as table_alias -where conflict_stream_name is not null --- conflict_stream_name at conflict_stream_name/conflict_stream_name; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql deleted file mode 100644 index 17e8df654774..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_hashid, - cast(conflict_stream_name as - string -) as conflict_stream_name, - _airbyte_emitted_at -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab1` --- conflict_stream_name at conflict_stream_name/conflict_stream_name; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql deleted file mode 100644 index 66e0c7cc4189..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(_airbyte_conflict_stream_name_hashid as - string -), ''), '-', coalesce(cast(conflict_stream_name as - string -), '')) as - string -))) as _airbyte_conflict_stream_name_2_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab2` tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql deleted file mode 100644 index 43ebdecf36c4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_2_hashid, - json_extract_scalar(conflict_stream_name, "$['groups']") as `groups`, - _airbyte_emitted_at -from `dataline-integration-testing`.test_normalization.`conflict_stream_name_conflict_stream_name` as table_alias -where conflict_stream_name is not null --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql deleted file mode 100644 index b1674b6fb1cd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_2_hashid, - cast(`groups` as - string -) as `groups`, - _airbyte_emitted_at -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1` --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql deleted file mode 100644 index 1fdb660bcfac..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as - string -), ''), '-', coalesce(cast(`groups` as - string -), '')) as - string -))) as _airbyte_conflict_stream_name_3_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2` tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql deleted file mode 100644 index bd1d66a5840f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ /dev/null @@ -1,13 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_scalar(_airbyte_data, "$['conflict_stream_scalar']") as conflict_stream_scalar, - _airbyte_emitted_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_scalar as table_alias --- conflict_stream_scalar; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql deleted file mode 100644 index 2a3441329550..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - string -) as id, - cast(conflict_stream_scalar as - int64 -) as conflict_stream_scalar, - _airbyte_emitted_at -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab1` --- conflict_stream_scalar; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql deleted file mode 100644 index 5aea888837e8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(conflict_stream_scalar as - string -), '')) as - string -))) as _airbyte_conflict_stream_scalar_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab2` tmp --- conflict_stream_scalar; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql deleted file mode 100644 index ecd6deee4576..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_scalar(_airbyte_data, "$['date']") as date, - - json_extract(table_alias._airbyte_data, "$['partition']") - as `partition`, - _airbyte_emitted_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias --- nested_stream_with_complex_columns_resulting_into_long_names; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql deleted file mode 100644 index ed49b9abf584..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - string -) as id, - cast(date as - string -) as date, - cast(`partition` as - string -) as `partition`, - _airbyte_emitted_at -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab1` --- nested_stream_with_complex_columns_resulting_into_long_names; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql deleted file mode 100644 index cdf282290087..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(date as - string -), ''), '-', coalesce(cast(`partition` as - string -), '')) as - string -))) as _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab2` tmp --- nested_stream_with_complex_columns_resulting_into_long_names; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab4.sql deleted file mode 100644 index fe9e5f7b18ea..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab4.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab4` - OPTIONS() - as --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab3` tmp --- nested_stream_with_complex_columns_resulting_into_long_names from `dataline-integration-testing`.test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql deleted file mode 100644 index ffc273b03241..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_partition_hashid, - json_extract_scalar(DATA, "$['currency']") as currency, - _airbyte_emitted_at -from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` as table_alias -cross join unnest(DATA) as DATA -where DATA is not null --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql deleted file mode 100644 index 31ee702ba3ec..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as - string -) as currency, - _airbyte_emitted_at -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1` --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql deleted file mode 100644 index 4a44b93291e8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as - string -), ''), '-', coalesce(cast(currency as - string -), '')) as - string -))) as _airbyte_DATA_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2` tmp --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql deleted file mode 100644 index fae58456c350..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, - json_extract_array(`partition`, "$['double_array_data']") as double_array_data, - json_extract_array(`partition`, "$['DATA']") as DATA, - json_extract_array(`partition`, "$['column___with__quotes']") as column___with__quotes, - _airbyte_emitted_at -from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names` as table_alias -where `partition` is not null --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql deleted file mode 100644 index a0eeb3ef422b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, - double_array_data, - DATA, - column___with__quotes, - _airbyte_emitted_at -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1` --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql deleted file mode 100644 index 99eba178cf0c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql +++ /dev/null @@ -1,22 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid as - string -), ''), '-', coalesce(cast(array_to_string(double_array_data, "|", "") as - string -), ''), '-', coalesce(cast(array_to_string(DATA, "|", "") as - string -), ''), '-', coalesce(cast(array_to_string(column___with__quotes, "|", "") as - string -), '')) as - string -))) as _airbyte_partition_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2` tmp --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql deleted file mode 100644 index 1e59ead141ba..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_partition_hashid, - json_extract_scalar(column___with__quotes, "$['currency']") as currency, - _airbyte_emitted_at -from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` as table_alias -cross join unnest(column___with__quotes) as column___with__quotes -where column___with__quotes is not null --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql deleted file mode 100644 index be9eef8648af..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as - string -) as currency, - _airbyte_emitted_at -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1` --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql deleted file mode 100644 index b76ca1ee8599..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as - string -), ''), '-', coalesce(cast(currency as - string -), '')) as - string -))) as _airbyte_column___with__quotes_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2` tmp --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql deleted file mode 100644 index bc13ba7f2783..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_partition_hashid, - json_extract_scalar(double_array_data, "$['id']") as id, - _airbyte_emitted_at -from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` as table_alias -cross join unnest(double_array_data) as double_array_data -where double_array_data is not null --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql deleted file mode 100644 index 8de95f382eb3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(id as - string -) as id, - _airbyte_emitted_at -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1` --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql deleted file mode 100644 index 19fca86225bc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as - string -), ''), '-', coalesce(cast(id as - string -), '')) as - string -))) as _airbyte_double_array_data_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2` tmp --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql deleted file mode 100644 index d00f9c167c3a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql +++ /dev/null @@ -1,13 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_scalar(_airbyte_data, "$['date']") as date, - _airbyte_emitted_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias --- non_nested_stream_without_namespace_resulting_into_long_names; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql deleted file mode 100644 index e653ba297d30..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - string -) as id, - cast(date as - string -) as date, - _airbyte_emitted_at -from `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab1` --- non_nested_stream_without_namespace_resulting_into_long_names; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql deleted file mode 100644 index 7b33b70adccb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(date as - string -), '')) as - string -))) as _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab2` tmp --- non_nested_stream_without_namespace_resulting_into_long_names; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab1.sql deleted file mode 100644 index 74d16273b1c1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ /dev/null @@ -1,13 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_array(_airbyte_data, "$['children']") as children, - _airbyte_emitted_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_unnest_alias as table_alias --- unnest_alias; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab2.sql deleted file mode 100644 index ff4c7a287bc3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - int64 -) as id, - children, - _airbyte_emitted_at -from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab1` --- unnest_alias; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab3.sql deleted file mode 100644 index 7adcbf0136c8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(array_to_string(children, "|", "") as - string -), '')) as - string -))) as _airbyte_unnest_alias_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab2` tmp --- unnest_alias; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql deleted file mode 100644 index 0dd96b131801..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_unnest_alias_hashid, - json_extract_scalar(children, "$['ab_id']") as ab_id, - - json_extract(children, "$['owner']") - as owner, - _airbyte_emitted_at -from `dataline-integration-testing`.test_normalization.`unnest_alias` as table_alias -cross join unnest(children) as children -where children is not null --- children at unnest_alias/children; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql deleted file mode 100644 index fc350690e2d4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_unnest_alias_hashid, - cast(ab_id as - int64 -) as ab_id, - cast(owner as - string -) as owner, - _airbyte_emitted_at -from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab1` --- children at unnest_alias/children; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql deleted file mode 100644 index db47bd2eac1e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(_airbyte_unnest_alias_hashid as - string -), ''), '-', coalesce(cast(ab_id as - string -), ''), '-', coalesce(cast(owner as - string -), '')) as - string -))) as _airbyte_children_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab2` tmp --- children at unnest_alias/children; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql deleted file mode 100644 index 7e66ffa11d0e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_children_hashid, - json_extract_scalar(owner, "$['owner_id']") as owner_id, - _airbyte_emitted_at -from `dataline-integration-testing`.test_normalization.`unnest_alias_children` as table_alias -where owner is not null --- owner at unnest_alias/children/owner; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql deleted file mode 100644 index 58a22b1b80d2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_children_hashid, - cast(owner_id as - int64 -) as owner_id, - _airbyte_emitted_at -from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab1` --- owner at unnest_alias/children/owner; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql deleted file mode 100644 index 2e2fffb17d68..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(_airbyte_children_hashid as - string -), ''), '-', coalesce(cast(owner_id as - string -), '')) as - string -))) as _airbyte_owner_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab2` tmp --- owner at unnest_alias/children/owner; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql deleted file mode 100644 index c7e36ed1112d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql +++ /dev/null @@ -1,13 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_scalar(_airbyte_data, "$['date']") as date, - _airbyte_emitted_at -from `dataline-integration-testing`.test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias --- simple_stream_with_namespace_resulting_into_long_names; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql deleted file mode 100644 index d7cebc4de520..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - string -) as id, - cast(date as - string -) as date, - _airbyte_emitted_at -from `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab1` --- simple_stream_with_namespace_resulting_into_long_names; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql deleted file mode 100644 index 2b005f0f853c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(date as - string -), '')) as - string -))) as _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab2` tmp --- simple_stream_with_namespace_resulting_into_long_names; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql deleted file mode 100644 index efb1c7223ca9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_array` - - - OPTIONS() - as ( - --- Final base SQL model -select - id, - conflict_stream_array, - _airbyte_emitted_at, - _airbyte_conflict_stream_array_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab3` --- conflict_stream_array from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_array - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql deleted file mode 100644 index c660adb89754..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_name` - - - OPTIONS() - as ( - --- Final base SQL model -select - id, - conflict_stream_name, - _airbyte_emitted_at, - _airbyte_conflict_stream_name_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab3` --- conflict_stream_name from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_name - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql deleted file mode 100644 index 1083f66f46a0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_name_conflict_stream_name` - - - OPTIONS() - as ( - --- Final base SQL model -select - _airbyte_conflict_stream_name_hashid, - conflict_stream_name, - _airbyte_emitted_at, - _airbyte_conflict_stream_name_2_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab3` --- conflict_stream_name at conflict_stream_name/conflict_stream_name from `dataline-integration-testing`.test_normalization.`conflict_stream_name` - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql deleted file mode 100644 index 573cfc4a54fb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name` - - - OPTIONS() - as ( - --- Final base SQL model -select - _airbyte_conflict_stream_name_2_hashid, - `groups`, - _airbyte_emitted_at, - _airbyte_conflict_stream_name_3_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3` --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from `dataline-integration-testing`.test_normalization.`conflict_stream_name_conflict_stream_name` - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql deleted file mode 100644 index 40652d62ec2c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_scalar` - - - OPTIONS() - as ( - --- Final base SQL model -select - id, - conflict_stream_scalar, - _airbyte_emitted_at, - _airbyte_conflict_stream_scalar_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab3` --- conflict_stream_scalar from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_scalar - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql deleted file mode 100644 index 721d03c1e503..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names` - - - OPTIONS() - as ( - --- Final base SQL model -select - id, - date, - `partition`, - _airbyte_emitted_at, - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid -from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_scd` --- nested_stream_with_complex_columns_resulting_into_long_names from `dataline-integration-testing`.test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names -where _airbyte_active_row = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql deleted file mode 100644 index c5cfbbd0b526..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` - - - OPTIONS() - as ( - --- Final base SQL model -select - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, - double_array_data, - DATA, - column___with__quotes, - _airbyte_emitted_at, - _airbyte_partition_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3` --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names` - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql deleted file mode 100644 index 42cacd429408..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA` - - - OPTIONS() - as ( - --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_emitted_at, - _airbyte_DATA_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3` --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql deleted file mode 100644 index ef3a1635a6e4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes` - - - OPTIONS() - as ( - --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_emitted_at, - _airbyte_column___with__quotes_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3` --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql deleted file mode 100644 index 088994fc6fc1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data` - - - OPTIONS() - as ( - --- Final base SQL model -select - _airbyte_partition_hashid, - id, - _airbyte_emitted_at, - _airbyte_double_array_data_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3` --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql deleted file mode 100644 index 70229a37cad3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql +++ /dev/null @@ -1,29 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_scd` - - - OPTIONS() - as ( - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - date, - `partition`, - date as _airbyte_start_at, - lag(date) over ( - partition by id - order by date is null asc, date desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(date) over ( - partition by id - order by date is null asc, date desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab4` --- nested_stream_with_complex_columns_resulting_into_long_names from `dataline-integration-testing`.test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names -where _airbyte_row_num = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql deleted file mode 100644 index 271d231a59f9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names` - - - OPTIONS() - as ( - --- Final base SQL model -select - id, - date, - _airbyte_emitted_at, - _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab3` --- non_nested_stream_without_namespace_resulting_into_long_names from `dataline-integration-testing`.test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql deleted file mode 100644 index 8b0cc5c7f1a1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`unnest_alias` - - - OPTIONS() - as ( - --- Final base SQL model -select - id, - children, - _airbyte_emitted_at, - _airbyte_unnest_alias_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab3` --- unnest_alias from `dataline-integration-testing`.test_normalization._airbyte_raw_unnest_alias - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql deleted file mode 100644 index e40743c250bf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`unnest_alias_children` - - - OPTIONS() - as ( - --- Final base SQL model -select - _airbyte_unnest_alias_hashid, - ab_id, - owner, - _airbyte_emitted_at, - _airbyte_children_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab3` --- children at unnest_alias/children from `dataline-integration-testing`.test_normalization.`unnest_alias` - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql deleted file mode 100644 index 7bc307d7269f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`unnest_alias_children_owner` - - - OPTIONS() - as ( - --- Final base SQL model -select - _airbyte_children_hashid, - owner_id, - _airbyte_emitted_at, - _airbyte_owner_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab3` --- owner at unnest_alias/children/owner from `dataline-integration-testing`.test_normalization.`unnest_alias_children` - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql deleted file mode 100644 index cf45f438b2ec..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names` - - - OPTIONS() - as ( - --- Final base SQL model -select - id, - date, - _airbyte_emitted_at, - _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid -from `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab3` --- simple_stream_with_namespace_resulting_into_long_names from `dataline-integration-testing`.test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql new file mode 100644 index 000000000000..877a01b4c107 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql @@ -0,0 +1,16 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_array(_airbyte_data, "$['conflict_stream_array']") as conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_array as table_alias +-- conflict_stream_array +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql new file mode 100644 index 000000000000..13acc814aa35 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + string +) as id, + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab1` +-- conflict_stream_array +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql new file mode 100644 index 000000000000..b0a10f06f6fe --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(array_to_string(conflict_stream_array, "|", "") as + string +), '')) as + string +))) as _airbyte_conflict_stream_array_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab2` tmp +-- conflict_stream_array +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..e32dbb75a795 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + + json_extract(table_alias._airbyte_data, "$['conflict_stream_name']") + as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_name as table_alias +-- conflict_stream_name +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..1e6066d334e2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql @@ -0,0 +1,20 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + string +) as id, + cast(conflict_stream_name as + string +) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab1` +-- conflict_stream_name +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..f8867d6b5713 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(conflict_stream_name as + string +), '')) as + string +))) as _airbyte_conflict_stream_name_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab2` tmp +-- conflict_stream_name +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..57cef056c421 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_hashid, + + json_extract(table_alias.conflict_stream_name, "$['conflict_stream_name']") + as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization.`conflict_stream_name` as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..eae8466d7725 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_hashid, + cast(conflict_stream_name as + string +) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab1` +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..0ea122e2ea01 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(_airbyte_conflict_stream_name_hashid as + string +), ''), '-', coalesce(cast(conflict_stream_name as + string +), '')) as + string +))) as _airbyte_conflict_stream_name_2_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab2` tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..962511141a3b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql @@ -0,0 +1,17 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_2_hashid, + json_extract_scalar(conflict_stream_name, "$['groups']") as `groups`, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization.`conflict_stream_name_conflict_stream_name` as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..be894b9a62a3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_2_hashid, + cast(`groups` as + string +) as `groups`, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1` +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..c468c76f6378 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as + string +), ''), '-', coalesce(cast(`groups` as + string +), '')) as + string +))) as _airbyte_conflict_stream_name_3_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2` tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql new file mode 100644 index 000000000000..c933d2c41fac --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql @@ -0,0 +1,16 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['conflict_stream_scalar']") as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_scalar as table_alias +-- conflict_stream_scalar +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql new file mode 100644 index 000000000000..f1c973037920 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql @@ -0,0 +1,20 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + string +) as id, + cast(conflict_stream_scalar as + int64 +) as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab1` +-- conflict_stream_scalar +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql new file mode 100644 index 000000000000..2679d90727ee --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(conflict_stream_scalar as + string +), '')) as + string +))) as _airbyte_conflict_stream_scalar_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab2` tmp +-- conflict_stream_scalar +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql new file mode 100644 index 000000000000..19a85c1c7d63 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql @@ -0,0 +1,20 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['date']") as date, + + json_extract(table_alias._airbyte_data, "$['partition']") + as `partition`, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql new file mode 100644 index 000000000000..a7d06c7427b8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql @@ -0,0 +1,24 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + string +) as id, + cast(date as + string +) as date, + cast(`partition` as + string +) as `partition`, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab1` +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql new file mode 100644 index 000000000000..a6fa2d3eba18 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + json_extract_scalar(DATA, "$['currency']") as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` as table_alias +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +cross join unnest(DATA) as DATA +where 1 = 1 +and DATA is not null; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql new file mode 100644 index 000000000000..1e4b82a39dec --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as + string +) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1` +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql new file mode 100644 index 000000000000..c60a413f2d05 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as + string +), ''), '-', coalesce(cast(currency as + string +), '')) as + string +))) as _airbyte_DATA_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2` tmp +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql new file mode 100644 index 000000000000..556ff2471ce9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + json_extract_array(`partition`, "$['double_array_data']") as double_array_data, + json_extract_array(`partition`, "$['DATA']") as DATA, + json_extract_array(`partition`, "$['column___with__quotes']") as column___with__quotes, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names` as table_alias +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and `partition` is not null; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql new file mode 100644 index 000000000000..0dbb6ecbb1f9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + double_array_data, + DATA, + column___with__quotes, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1` +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql new file mode 100644 index 000000000000..56fd4a34bd7f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql @@ -0,0 +1,23 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid as + string +), ''), '-', coalesce(cast(array_to_string(double_array_data, "|", "") as + string +), ''), '-', coalesce(cast(array_to_string(DATA, "|", "") as + string +), ''), '-', coalesce(cast(array_to_string(column___with__quotes, "|", "") as + string +), '')) as + string +))) as _airbyte_partition_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2` tmp +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql new file mode 100644 index 000000000000..e7ab56070330 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + json_extract_scalar(column___with__quotes, "$['currency']") as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` as table_alias +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +cross join unnest(column___with__quotes) as column___with__quotes +where 1 = 1 +and column___with__quotes is not null; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql new file mode 100644 index 000000000000..b552081271d7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as + string +) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1` +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql new file mode 100644 index 000000000000..cdc26222cfab --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as + string +), ''), '-', coalesce(cast(currency as + string +), '')) as + string +))) as _airbyte_column___with__quotes_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2` tmp +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql new file mode 100644 index 000000000000..18469ff3ada9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + json_extract_scalar(double_array_data, "$['id']") as id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` as table_alias +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +cross join unnest(double_array_data) as double_array_data +where 1 = 1 +and double_array_data is not null; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql new file mode 100644 index 000000000000..bc3564b761f1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(id as + string +) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1` +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql new file mode 100644 index 000000000000..410d26ac7a07 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as + string +), ''), '-', coalesce(cast(id as + string +), '')) as + string +))) as _airbyte_double_array_data_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2` tmp +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql new file mode 100644 index 000000000000..eae11107c950 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql @@ -0,0 +1,16 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['date']") as date, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias +-- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql new file mode 100644 index 000000000000..3006d17a73e9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql @@ -0,0 +1,20 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + string +) as id, + cast(date as + string +) as date, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab1` +-- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql new file mode 100644 index 000000000000..707db214afcd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(date as + string +), '')) as + string +))) as _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab2` tmp +-- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql new file mode 100644 index 000000000000..043cd72e489d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql @@ -0,0 +1,17 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_array(_airbyte_data, "$['children']") as children, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_unnest_alias as table_alias +-- unnest_alias +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql new file mode 100644 index 000000000000..e99a33441a66 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + int64 +) as id, + children, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab1` +-- unnest_alias +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql new file mode 100644 index 000000000000..d807cce078ad --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql @@ -0,0 +1,20 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(array_to_string(children, "|", "") as + string +), '')) as + string +))) as _airbyte_unnest_alias_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab2` tmp +-- unnest_alias +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql new file mode 100644 index 000000000000..9896f675369f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql @@ -0,0 +1,22 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_unnest_alias_hashid, + json_extract_scalar(children, "$['ab_id']") as ab_id, + + json_extract(children, "$['owner']") + as owner, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization.`unnest_alias` as table_alias +-- children at unnest_alias/children +cross join unnest(children) as children +where 1 = 1 +and children is not null; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql new file mode 100644 index 000000000000..4347ac1d1e98 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql @@ -0,0 +1,21 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_unnest_alias_hashid, + cast(ab_id as + int64 +) as ab_id, + cast(owner as + string +) as owner, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab1` +-- children at unnest_alias/children +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql new file mode 100644 index 000000000000..07b1fea81aa2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql @@ -0,0 +1,21 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(_airbyte_unnest_alias_hashid as + string +), ''), '-', coalesce(cast(ab_id as + string +), ''), '-', coalesce(cast(owner as + string +), '')) as + string +))) as _airbyte_children_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab2` tmp +-- children at unnest_alias/children +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql new file mode 100644 index 000000000000..0af940c468cf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql @@ -0,0 +1,17 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_children_hashid, + json_extract_scalar(owner, "$['owner_id']") as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization.`unnest_alias_children` as table_alias +-- owner at unnest_alias/children/owner +where 1 = 1 +and owner is not null; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql new file mode 100644 index 000000000000..1d59670f9820 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_children_hashid, + cast(owner_id as + int64 +) as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab1` +-- owner at unnest_alias/children/owner +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql new file mode 100644 index 000000000000..4c1f50bc18a4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(_airbyte_children_hashid as + string +), ''), '-', coalesce(cast(owner_id as + string +), '')) as + string +))) as _airbyte_owner_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab2` tmp +-- owner at unnest_alias/children/owner +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql new file mode 100644 index 000000000000..7d71f4199706 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql @@ -0,0 +1,16 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['date']") as date, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias +-- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql new file mode 100644 index 000000000000..51c4df2f7fba --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql @@ -0,0 +1,20 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + string +) as id, + cast(date as + string +) as date, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab1` +-- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql new file mode 100644 index 000000000000..b86eed32466f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(date as + string +), '')) as + string +))) as _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab2` tmp +-- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql new file mode 100644 index 000000000000..11c5f0e7d7c9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -0,0 +1,86 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_scd` + partition by range_bucket( + _airbyte_active_row, + generate_array(0, 1, 1) + ) + cluster by _airbyte_unique_key, _airbyte_emitted_at + OPTIONS() + as ( + +with + +input_data as ( + select * + from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab3` + -- nested_stream_with_complex_columns_resulting_into_long_names from `dataline-integration-testing`.test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), '')) as + string +))) as _airbyte_unique_key, + id, + date, + `partition`, + date as _airbyte_start_at, + lag(date) over ( + partition by id + order by + date is null asc, + date desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(date) over ( + partition by id + order by + date is null asc, + date desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + to_hex(md5(cast(concat(coalesce(cast(_airbyte_unique_key as + string +), ''), '-', coalesce(cast(_airbyte_start_at as + string +), ''), '-', coalesce(cast(_airbyte_emitted_at as + string +), '')) as + string +))) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + date, + `partition`, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid +from dedup_data where _airbyte_row_num = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql new file mode 100644 index 000000000000..17d17751b157 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql @@ -0,0 +1,25 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_unique_key, _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + date, + `partition`, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid +from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_scd` +-- nested_stream_with_complex_columns_resulting_into_long_names from `dataline-integration-testing`.test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 +and _airbyte_active_row = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..a3e70d30888d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql @@ -0,0 +1,22 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`unnest_alias` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + id, + children, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_unnest_alias_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab3` +-- unnest_alias from `dataline-integration-testing`.test_normalization._airbyte_raw_unnest_alias +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql new file mode 100644 index 000000000000..f641bb360a1b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_array` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + id, + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_conflict_stream_array_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab3` +-- conflict_stream_array from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_array +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql new file mode 100644 index 000000000000..2a7669945831 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_name` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + id, + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab3` +-- conflict_stream_name from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_name +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql new file mode 100644 index 000000000000..1d897d05e549 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_name_conflict_stream_name` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_conflict_stream_name_hashid, + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_2_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab3` +-- conflict_stream_name at conflict_stream_name/conflict_stream_name from `dataline-integration-testing`.test_normalization.`conflict_stream_name` +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql new file mode 100644 index 000000000000..033f337977b9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_conflict_stream_name_2_hashid, + `groups`, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_3_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3` +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from `dataline-integration-testing`.test_normalization.`conflict_stream_name_conflict_stream_name` +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql new file mode 100644 index 000000000000..b5b99cd91982 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_scalar` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + id, + conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_conflict_stream_scalar_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab3` +-- conflict_stream_scalar from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_scalar +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql new file mode 100644 index 000000000000..9e5aca960cfd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql @@ -0,0 +1,23 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + double_array_data, + DATA, + column___with__quotes, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_partition_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3` +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names` +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql new file mode 100644 index 000000000000..d8f8943f5149 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_DATA_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3` +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql new file mode 100644 index 000000000000..01a20f04da09 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_column___with__quotes_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3` +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql new file mode 100644 index 000000000000..639fa690b35b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_partition_hashid, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_double_array_data_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3` +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql new file mode 100644 index 000000000000..880063feec3c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + id, + date, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab3` +-- non_nested_stream_without_namespace_resulting_into_long_names from `dataline-integration-testing`.test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql new file mode 100644 index 000000000000..b0aa1b41329f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -0,0 +1,22 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`unnest_alias_children` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_unnest_alias_hashid, + ab_id, + owner, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_children_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab3` +-- children at unnest_alias/children from `dataline-integration-testing`.test_normalization.`unnest_alias` +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql new file mode 100644 index 000000000000..4be0121256bd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`unnest_alias_children_owner` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_children_hashid, + owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_owner_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab3` +-- owner at unnest_alias/children/owner from `dataline-integration-testing`.test_normalization.`unnest_alias_children` +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql new file mode 100644 index 000000000000..b2236adf2322 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + id, + date, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid +from `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab3` +-- simple_stream_with_namespace_resulting_into_long_names from `dataline-integration-testing`.test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql new file mode 100644 index 000000000000..36e9afe3d413 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql @@ -0,0 +1,22 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(date as + string +), ''), '-', coalesce(cast(`partition` as + string +), '')) as + string +))) as _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab2` tmp +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql deleted file mode 100644 index f724978e5d33..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql +++ /dev/null @@ -1,19 +0,0 @@ -with table_row_counts as ( - select distinct count(*) as row_count, 2 as expected_count - from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} -union all - select distinct count(*) as row_count, 2 as expected_count - from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names') }} -union all - select distinct count(*) as row_count, 2 as expected_count - from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} -union all - select count(distinct currency) as row_count, 1 as expected_count - from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA') }} --- union all --- select count(distinct id) as row_count, 3 as expected_count --- from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data') }} -) -select * -from table_row_counts -where row_count != expected_count diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql index d3df56ab3d35..17c06b46b9de 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_array('_airbyte_data', ['conflict_stream_array'], ['conflict_stream_array']) }} as conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} as table_alias -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql index 2f0da1bb06e7..b9f5ca00d5f1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_array_ab1') }} -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql index 0a92c655f92a..925451397e03 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +14,5 @@ select tmp.* from {{ ref('conflict_stream_array_ab2') }} tmp -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql index 8556d7f50afb..850e03194fc5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract('table_alias', '_airbyte_data', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} as table_alias -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql index a1d77f379bce..1d6d46e0d1ff 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_ab1') }} -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql index dc562af89ba1..1ea9d02a7136 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +14,5 @@ select tmp.* from {{ ref('conflict_stream_name_ab2') }} tmp -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql index d6e2edb75ced..e6c77c8bc197 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql @@ -1,10 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_conflict_stream_name_hashid, {{ json_extract('table_alias', 'conflict_stream_name', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name') }} as table_alias -where conflict_stream_name is not null -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql index 463a878933f8..80bebf6d45b6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_conflict_stream_name_hashid, cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_conflict_stream_name_ab1') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql index f8d96123ce25..eaec60ab1634 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +14,5 @@ select tmp.* from {{ ref('conflict_stream_name_conflict_stream_name_ab2') }} tmp -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql index e29eff6b3c01..917c7f8d18ca 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql @@ -1,10 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_conflict_stream_name_2_hashid, {{ json_extract_scalar('conflict_stream_name', ['groups'], ['groups']) }} as {{ adapter.quote('groups') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_conflict_stream_name') }} as table_alias -where conflict_stream_name is not null -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql index 759e3a18d132..f20eec57368b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_conflict_stream_name_2_hashid, cast({{ adapter.quote('groups') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('groups') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql index e05638772e8d..67fab3b94f9e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +14,5 @@ select tmp.* from {{ ref('conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2') }} tmp -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql index be57970e399e..eb1e66fa8ee3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['conflict_stream_scalar'], ['conflict_stream_scalar']) }} as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} as table_alias -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql index cf0d9deb51cd..b66d0d244bec 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast(conflict_stream_scalar as {{ dbt_utils.type_bigint() }}) as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_scalar_ab1') }} -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql index 781c4bbfc391..18e6e00c4681 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +14,5 @@ select tmp.* from {{ ref('conflict_stream_scalar_ab2') }} tmp -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql index aaa4eb63f243..2f9195fcd8f1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql @@ -1,10 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as date, {{ json_extract('table_alias', '_airbyte_data', ['partition'], ['partition']) }} as {{ adapter.quote('partition') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} as table_alias -- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql index 63bd774a716f..d8f527d720d4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql @@ -1,10 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast(date as {{ dbt_utils.type_string() }}) as date, cast({{ adapter.quote('partition') }} as {{ type_json() }}) as {{ adapter.quote('partition') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab1') }} -- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql deleted file mode 100644 index 23ffaa321bd8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql +++ /dev/null @@ -1,12 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'date', - adapter.quote('partition'), - ]) }} as _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, - tmp.* -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab2') }} tmp --- nested_stream_with_complex_columns_resulting_into_long_names - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab4.sql deleted file mode 100644 index e2c9a1304fde..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab3') }} tmp --- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql index 055fa0fd6186..1e77394f833f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql @@ -1,12 +1,21 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_complex_columns_resulting_into_long_names_partition', 'partition', 'DATA') }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value('DATA'), ['currency'], ['currency']) }} as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', 'DATA') }} -where DATA is not null -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +{{ cross_join_unnest('partition', 'DATA') }} +where 1 = 1 +and DATA is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql index be7b0d9e999d..d069f1a86923 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(currency as {{ dbt_utils.type_string() }}) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1') }} -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql index 8141858e0533..856be375a7cc 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +14,5 @@ select tmp.* from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2') }} tmp -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql index dbe34c46db5e..b5cd1da52bd6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql @@ -1,12 +1,21 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, {{ json_extract_array(adapter.quote('partition'), ['double_array_data'], ['double_array_data']) }} as double_array_data, {{ json_extract_array(adapter.quote('partition'), ['DATA'], ['DATA']) }} as DATA, {{ json_extract_array(adapter.quote('partition'), ['column`_\'with"_quotes'], ['column___with__quotes']) }} as column___with__quotes, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names') }} as table_alias -where {{ adapter.quote('partition') }} is not null -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and {{ adapter.quote('partition') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql index 7c2087a397f4..23d645909627 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql @@ -1,11 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, double_array_data, DATA, column___with__quotes, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1') }} -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql index 101b256b8bb5..c77aba11220e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -10,4 +16,5 @@ select tmp.* from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2') }} tmp -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql index 4202a1cef1c6..4cbc027f3f71 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql @@ -1,12 +1,21 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_complex_columns_resulting_into_long_names_partition', 'partition', 'column___with__quotes') }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value('column___with__quotes'), ['currency'], ['currency']) }} as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', 'column___with__quotes') }} -where column___with__quotes is not null -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +{{ cross_join_unnest('partition', 'column___with__quotes') }} +where 1 = 1 +and column___with__quotes is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql index 076b4bfc975c..8a107f04e7e8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(currency as {{ dbt_utils.type_string() }}) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1') }} -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql index d4d5028ad31d..59b1e25f08fb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +14,5 @@ select tmp.* from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2') }} tmp -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql index 48b7ea236ae1..e7e798b05822 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql @@ -1,12 +1,21 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_complex_columns_resulting_into_long_names_partition', 'partition', 'double_array_data') }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value('double_array_data'), ['id'], ['id']) }} as id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', 'double_array_data') }} -where double_array_data is not null -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +{{ cross_join_unnest('partition', 'double_array_data') }} +where 1 = 1 +and double_array_data is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql index a3e29e92dfb7..034a47f9c894 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(id as {{ dbt_utils.type_string() }}) as id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1') }} -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql index 13295b47f97f..60a4c34970ac 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +14,5 @@ select tmp.* from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2') }} tmp -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql index fd2fbdd54863..ed2d3cd69d1d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as date, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names') }} as table_alias -- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql index e4ed8d075d22..671850765c83 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast(date as {{ dbt_utils.type_string() }}) as date, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('non_nested_stream_without_namespace_resulting_into_long_names_ab1') }} -- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql index cd80649af632..15f30e6226cf 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +14,5 @@ select tmp.* from {{ ref('non_nested_stream_without_namespace_resulting_into_long_names_ab2') }} tmp -- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql index 15b34c1ca7d4..5a136fe032bb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql @@ -1,9 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_array('_airbyte_data', ['children'], ['children']) }} as children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} as table_alias -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql index a416538b5422..afc783083e6c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql @@ -1,9 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_ab1') }} -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql index 0c6703933b10..ce3a0718103d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +14,6 @@ select tmp.* from {{ ref('unnest_alias_ab2') }} tmp -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql index 57abeb23159f..995c1a2bdb48 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql @@ -1,13 +1,22 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('unnest_alias', 'unnest_alias', 'children') }} select _airbyte_unnest_alias_hashid, {{ json_extract_scalar(unnested_column_value('children'), ['ab_id'], ['ab_id']) }} as ab_id, {{ json_extract('', unnested_column_value('children'), ['owner'], ['owner']) }} as owner, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias') }} as table_alias -{{ cross_join_unnest('unnest_alias', 'children') }} -where children is not null -- children at unnest_alias/children +{{ cross_join_unnest('unnest_alias', 'children') }} +where 1 = 1 +and children is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql index 8db853c1d601..0cb0d82d585c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql @@ -1,10 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_unnest_alias_hashid, cast(ab_id as {{ dbt_utils.type_bigint() }}) as ab_id, cast(owner as {{ type_json() }}) as owner, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children_ab1') }} -- children at unnest_alias/children +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql index 9d84f3be6aa0..5110b59063a1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -9,4 +15,5 @@ select tmp.* from {{ ref('unnest_alias_children_ab2') }} tmp -- children at unnest_alias/children +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql index 4f28c4f8b5cb..edda3677624d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql @@ -1,10 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_children_hashid, {{ json_extract_scalar('owner', ['owner_id'], ['owner_id']) }} as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children') }} as table_alias -where owner is not null -- owner at unnest_alias/children/owner +where 1 = 1 +and owner is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql index a0f474c62db7..5bc8a3f96b2c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_children_hashid, cast(owner_id as {{ dbt_utils.type_bigint() }}) as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children_owner_ab1') }} -- owner at unnest_alias/children/owner +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql index c5f6562c34c2..16753dc228c5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +14,5 @@ select tmp.* from {{ ref('unnest_alias_children_owner_ab2') }} tmp -- owner at unnest_alias/children/owner +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql index e3ec7ffd2f82..f29ea4161b7d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as date, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization_namespace', '_airbyte_raw_simple_stream_with_namespace_resulting_into_long_names') }} as table_alias -- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql index c437ca698089..11e2b2ec37bd 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast(date as {{ dbt_utils.type_string() }}) as date, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('simple_stream_with_namespace_resulting_into_long_names_ab1') }} -- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql index 727eabab6ac2..79e6653964c0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +14,5 @@ select tmp.* from {{ ref('simple_stream_with_namespace_resulting_into_long_names_ab2') }} tmp -- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql new file mode 100644 index 000000000000..b2389c8fd05d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -0,0 +1,109 @@ +{{ config( + cluster_by = ["_airbyte_unique_key","_airbyte_emitted_at"], + partition_by = {"field": "_airbyte_active_row", "data_type": "int64", "range": {"start": 0, "end": 1, "interval": 1}}, + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab3') }} + -- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('nested_stream_with_complex_columns_resulting_into_long_names_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('nested_stream_with_complex_columns_resulting_into_long_names_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('nested_stream_with_complex_columns_resulting_into_long_names_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab3') }} + -- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + date, + {{ adapter.quote('partition') }}, + date as _airbyte_start_at, + lag(date) over ( + partition by id + order by + date is null asc, + date desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(date) over ( + partition by id + order by + date is null asc, + date desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + date, + {{ adapter.quote('partition') }}, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql new file mode 100644 index 000000000000..308499910d87 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql @@ -0,0 +1,23 @@ +{{ config( + cluster_by = ["_airbyte_unique_key","_airbyte_emitted_at"], + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + id, + date, + {{ adapter.quote('partition') }}, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid +from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_scd') }} +-- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..1c38bf472728 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql @@ -0,0 +1,20 @@ +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + id, + children, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_unnest_alias_hashid +from {{ ref('unnest_alias_ab3') }} +-- unnest_alias from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql index 9ffda6086e35..fd790b1b5f1a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -1,10 +1,19 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, conflict_stream_array, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_array_hashid from {{ ref('conflict_stream_array_ab3') }} -- conflict_stream_array from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql index 8a83ccc01e31..def7986d8580 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -1,10 +1,19 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_hashid from {{ ref('conflict_stream_name_ab3') }} -- conflict_stream_name from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql index 94a9dc3d0139..539ff778ebdf 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -1,10 +1,19 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_conflict_stream_name_hashid, conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_2_hashid from {{ ref('conflict_stream_name_conflict_stream_name_ab3') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql index 36c23b81b82d..1f43352fa617 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql @@ -1,10 +1,19 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_conflict_stream_name_2_hashid, {{ adapter.quote('groups') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_3_hashid from {{ ref('conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name_conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql index 9d7acc846fd0..a013b69e57aa 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -1,10 +1,19 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, conflict_stream_scalar, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_scalar_hashid from {{ ref('conflict_stream_scalar_ab3') }} -- conflict_stream_scalar from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql deleted file mode 100644 index 45aafa818f6f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql +++ /dev/null @@ -1,12 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - date, - {{ adapter.quote('partition') }}, - _airbyte_emitted_at, - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_scd') }} --- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} -where _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql index 8461ffb2cabd..ef861e9fa13e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql @@ -1,12 +1,21 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, double_array_data, DATA, column___with__quotes, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_partition_hashid from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3') }} -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql index 4e7bf4841bd5..7b461f4989e8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql @@ -1,10 +1,19 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_DATA_hashid from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3') }} -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql index 2bd228318b0d..2bcc0ec32e51 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql @@ -1,10 +1,19 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_column___with__quotes_hashid from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3') }} -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql index 50b39ee05b38..353362155623 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql @@ -1,10 +1,19 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, id, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_double_array_data_hashid from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3') }} -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql deleted file mode 100644 index 06311da05fc6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - date, - {{ adapter.quote('partition') }}, - date as _airbyte_start_at, - lag(date) over ( - partition by id - order by date is null asc, date desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(date) over ( - partition by id - order by date is null asc, date desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab4') }} --- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql index 1004a9e2f4a4..14c598a1568d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql @@ -1,10 +1,19 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, date, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid from {{ ref('non_nested_stream_without_namespace_resulting_into_long_names_ab3') }} -- non_nested_stream_without_namespace_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql deleted file mode 100644 index 7a570c3836cb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,10 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - children, - _airbyte_emitted_at, - _airbyte_unnest_alias_hashid -from {{ ref('unnest_alias_ab3') }} --- unnest_alias from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql index 77b2b0ad854e..7e78f023a44e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -1,11 +1,20 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_unnest_alias_hashid, ab_id, owner, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_children_hashid from {{ ref('unnest_alias_children_ab3') }} -- children at unnest_alias/children from {{ ref('unnest_alias') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql index 0ccf8e33d8d6..c81e06c620b1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -1,10 +1,19 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_children_hashid, owner_id, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_owner_hashid from {{ ref('unnest_alias_children_owner_ab3') }} -- owner at unnest_alias/children/owner from {{ ref('unnest_alias_children') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql index ce74645f7864..54dec00c79ee 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql @@ -1,10 +1,19 @@ -{{ config(schema="test_normalization_namespace", tags=["top-level"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization_namespace", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, date, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid from {{ ref('simple_stream_with_namespace_resulting_into_long_names_ab3') }} -- simple_stream_with_namespace_resulting_into_long_names from {{ source('test_normalization_namespace', '_airbyte_raw_simple_stream_with_namespace_resulting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql new file mode 100644 index 000000000000..f4835b370e3c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql @@ -0,0 +1,20 @@ +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + 'date', + adapter.quote('partition'), + ]) }} as _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + tmp.* +from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab2') }} tmp +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql new file mode 100644 index 000000000000..877a01b4c107 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql @@ -0,0 +1,16 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_array(_airbyte_data, "$['conflict_stream_array']") as conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_array as table_alias +-- conflict_stream_array +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql new file mode 100644 index 000000000000..13acc814aa35 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + string +) as id, + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab1` +-- conflict_stream_array +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql new file mode 100644 index 000000000000..b0a10f06f6fe --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(array_to_string(conflict_stream_array, "|", "") as + string +), '')) as + string +))) as _airbyte_conflict_stream_array_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab2` tmp +-- conflict_stream_array +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..e32dbb75a795 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + + json_extract(table_alias._airbyte_data, "$['conflict_stream_name']") + as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_name as table_alias +-- conflict_stream_name +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..1e6066d334e2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql @@ -0,0 +1,20 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + string +) as id, + cast(conflict_stream_name as + string +) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab1` +-- conflict_stream_name +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..f8867d6b5713 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(conflict_stream_name as + string +), '')) as + string +))) as _airbyte_conflict_stream_name_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab2` tmp +-- conflict_stream_name +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..57cef056c421 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_hashid, + + json_extract(table_alias.conflict_stream_name, "$['conflict_stream_name']") + as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization.`conflict_stream_name` as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..eae8466d7725 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_hashid, + cast(conflict_stream_name as + string +) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab1` +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..0ea122e2ea01 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(_airbyte_conflict_stream_name_hashid as + string +), ''), '-', coalesce(cast(conflict_stream_name as + string +), '')) as + string +))) as _airbyte_conflict_stream_name_2_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab2` tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..962511141a3b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql @@ -0,0 +1,17 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_2_hashid, + json_extract_scalar(conflict_stream_name, "$['groups']") as `groups`, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization.`conflict_stream_name_conflict_stream_name` as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..be894b9a62a3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_2_hashid, + cast(`groups` as + string +) as `groups`, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1` +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..c468c76f6378 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as + string +), ''), '-', coalesce(cast(`groups` as + string +), '')) as + string +))) as _airbyte_conflict_stream_name_3_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2` tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql new file mode 100644 index 000000000000..c933d2c41fac --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql @@ -0,0 +1,16 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['conflict_stream_scalar']") as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_scalar as table_alias +-- conflict_stream_scalar +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql new file mode 100644 index 000000000000..f1c973037920 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql @@ -0,0 +1,20 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + string +) as id, + cast(conflict_stream_scalar as + int64 +) as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab1` +-- conflict_stream_scalar +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql new file mode 100644 index 000000000000..2679d90727ee --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(conflict_stream_scalar as + string +), '')) as + string +))) as _airbyte_conflict_stream_scalar_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab2` tmp +-- conflict_stream_scalar +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql new file mode 100644 index 000000000000..19a85c1c7d63 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql @@ -0,0 +1,20 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['date']") as date, + + json_extract(table_alias._airbyte_data, "$['partition']") + as `partition`, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql new file mode 100644 index 000000000000..a7d06c7427b8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql @@ -0,0 +1,24 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + string +) as id, + cast(date as + string +) as date, + cast(`partition` as + string +) as `partition`, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab1` +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql new file mode 100644 index 000000000000..a6fa2d3eba18 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + json_extract_scalar(DATA, "$['currency']") as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` as table_alias +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +cross join unnest(DATA) as DATA +where 1 = 1 +and DATA is not null; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql new file mode 100644 index 000000000000..1e4b82a39dec --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as + string +) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1` +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql new file mode 100644 index 000000000000..c60a413f2d05 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as + string +), ''), '-', coalesce(cast(currency as + string +), '')) as + string +))) as _airbyte_DATA_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2` tmp +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql new file mode 100644 index 000000000000..556ff2471ce9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + json_extract_array(`partition`, "$['double_array_data']") as double_array_data, + json_extract_array(`partition`, "$['DATA']") as DATA, + json_extract_array(`partition`, "$['column___with__quotes']") as column___with__quotes, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names` as table_alias +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and `partition` is not null; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql new file mode 100644 index 000000000000..0dbb6ecbb1f9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + double_array_data, + DATA, + column___with__quotes, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1` +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql new file mode 100644 index 000000000000..56fd4a34bd7f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql @@ -0,0 +1,23 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid as + string +), ''), '-', coalesce(cast(array_to_string(double_array_data, "|", "") as + string +), ''), '-', coalesce(cast(array_to_string(DATA, "|", "") as + string +), ''), '-', coalesce(cast(array_to_string(column___with__quotes, "|", "") as + string +), '')) as + string +))) as _airbyte_partition_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2` tmp +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql new file mode 100644 index 000000000000..e7ab56070330 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + json_extract_scalar(column___with__quotes, "$['currency']") as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` as table_alias +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +cross join unnest(column___with__quotes) as column___with__quotes +where 1 = 1 +and column___with__quotes is not null; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql new file mode 100644 index 000000000000..b552081271d7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as + string +) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1` +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql new file mode 100644 index 000000000000..cdc26222cfab --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as + string +), ''), '-', coalesce(cast(currency as + string +), '')) as + string +))) as _airbyte_column___with__quotes_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2` tmp +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql new file mode 100644 index 000000000000..18469ff3ada9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + json_extract_scalar(double_array_data, "$['id']") as id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` as table_alias +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +cross join unnest(double_array_data) as double_array_data +where 1 = 1 +and double_array_data is not null; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql new file mode 100644 index 000000000000..bc3564b761f1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(id as + string +) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1` +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql new file mode 100644 index 000000000000..410d26ac7a07 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as + string +), ''), '-', coalesce(cast(id as + string +), '')) as + string +))) as _airbyte_double_array_data_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2` tmp +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql new file mode 100644 index 000000000000..eae11107c950 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql @@ -0,0 +1,16 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['date']") as date, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias +-- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql new file mode 100644 index 000000000000..3006d17a73e9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql @@ -0,0 +1,20 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + string +) as id, + cast(date as + string +) as date, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab1` +-- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql new file mode 100644 index 000000000000..707db214afcd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(date as + string +), '')) as + string +))) as _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab2` tmp +-- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql new file mode 100644 index 000000000000..043cd72e489d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql @@ -0,0 +1,17 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_array(_airbyte_data, "$['children']") as children, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_unnest_alias as table_alias +-- unnest_alias +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql new file mode 100644 index 000000000000..e99a33441a66 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + int64 +) as id, + children, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab1` +-- unnest_alias +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql new file mode 100644 index 000000000000..d807cce078ad --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql @@ -0,0 +1,20 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(array_to_string(children, "|", "") as + string +), '')) as + string +))) as _airbyte_unnest_alias_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab2` tmp +-- unnest_alias +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql new file mode 100644 index 000000000000..9896f675369f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql @@ -0,0 +1,22 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_unnest_alias_hashid, + json_extract_scalar(children, "$['ab_id']") as ab_id, + + json_extract(children, "$['owner']") + as owner, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization.`unnest_alias` as table_alias +-- children at unnest_alias/children +cross join unnest(children) as children +where 1 = 1 +and children is not null; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql new file mode 100644 index 000000000000..4347ac1d1e98 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql @@ -0,0 +1,21 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_unnest_alias_hashid, + cast(ab_id as + int64 +) as ab_id, + cast(owner as + string +) as owner, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab1` +-- children at unnest_alias/children +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql new file mode 100644 index 000000000000..07b1fea81aa2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql @@ -0,0 +1,21 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(_airbyte_unnest_alias_hashid as + string +), ''), '-', coalesce(cast(ab_id as + string +), ''), '-', coalesce(cast(owner as + string +), '')) as + string +))) as _airbyte_children_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab2` tmp +-- children at unnest_alias/children +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql new file mode 100644 index 000000000000..0af940c468cf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql @@ -0,0 +1,17 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_children_hashid, + json_extract_scalar(owner, "$['owner_id']") as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization.`unnest_alias_children` as table_alias +-- owner at unnest_alias/children/owner +where 1 = 1 +and owner is not null; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql new file mode 100644 index 000000000000..1d59670f9820 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_children_hashid, + cast(owner_id as + int64 +) as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab1` +-- owner at unnest_alias/children/owner +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql new file mode 100644 index 000000000000..4c1f50bc18a4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(_airbyte_children_hashid as + string +), ''), '-', coalesce(cast(owner_id as + string +), '')) as + string +))) as _airbyte_owner_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab2` tmp +-- owner at unnest_alias/children/owner +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql new file mode 100644 index 000000000000..7d71f4199706 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql @@ -0,0 +1,16 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['date']") as date, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias +-- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql new file mode 100644 index 000000000000..51c4df2f7fba --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql @@ -0,0 +1,20 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + string +) as id, + cast(date as + string +) as date, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab1` +-- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql new file mode 100644 index 000000000000..b86eed32466f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(date as + string +), '')) as + string +))) as _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab2` tmp +-- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql new file mode 100644 index 000000000000..e2187e231d38 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -0,0 +1,27 @@ + + + + + + + + merge into `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_scd` as DBT_INTERNAL_DEST + using ( + select * from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_scd__dbt_tmp` + ) as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._airbyte_unique_key_scd = DBT_INTERNAL_DEST._airbyte_unique_key_scd + + + + when matched then update set + `_airbyte_unique_key` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key`,`_airbyte_unique_key_scd` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key_scd`,`id` = DBT_INTERNAL_SOURCE.`id`,`date` = DBT_INTERNAL_SOURCE.`date`,`partition` = DBT_INTERNAL_SOURCE.`partition`,`_airbyte_start_at` = DBT_INTERNAL_SOURCE.`_airbyte_start_at`,`_airbyte_end_at` = DBT_INTERNAL_SOURCE.`_airbyte_end_at`,`_airbyte_active_row` = DBT_INTERNAL_SOURCE.`_airbyte_active_row`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid` + + + when not matched then insert + (`_airbyte_unique_key`, `_airbyte_unique_key_scd`, `id`, `date`, `partition`, `_airbyte_start_at`, `_airbyte_end_at`, `_airbyte_active_row`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid`) + values + (`_airbyte_unique_key`, `_airbyte_unique_key_scd`, `id`, `date`, `partition`, `_airbyte_start_at`, `_airbyte_end_at`, `_airbyte_active_row`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid`) + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql new file mode 100644 index 000000000000..c2f7397d2c3b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql @@ -0,0 +1,27 @@ + + + + + + + + merge into `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names` as DBT_INTERNAL_DEST + using ( + select * from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names__dbt_tmp` + ) as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._airbyte_unique_key = DBT_INTERNAL_DEST._airbyte_unique_key + + + + when matched then update set + `_airbyte_unique_key` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key`,`id` = DBT_INTERNAL_SOURCE.`id`,`date` = DBT_INTERNAL_SOURCE.`date`,`partition` = DBT_INTERNAL_SOURCE.`partition`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid` + + + when not matched then insert + (`_airbyte_unique_key`, `id`, `date`, `partition`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid`) + values + (`_airbyte_unique_key`, `id`, `date`, `partition`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid`) + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..683fbd323fdb --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql @@ -0,0 +1,27 @@ + + + + + + + + merge into `dataline-integration-testing`.test_normalization.`unnest_alias` as DBT_INTERNAL_DEST + using ( + select * from `dataline-integration-testing`.test_normalization.`unnest_alias__dbt_tmp` + ) as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._airbyte_ab_id = DBT_INTERNAL_DEST._airbyte_ab_id + + + + when matched then update set + `id` = DBT_INTERNAL_SOURCE.`id`,`children` = DBT_INTERNAL_SOURCE.`children`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_unnest_alias_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_unnest_alias_hashid` + + + when not matched then insert + (`id`, `children`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_unnest_alias_hashid`) + values + (`id`, `children`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_unnest_alias_hashid`) + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql new file mode 100644 index 000000000000..f641bb360a1b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_array` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + id, + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_conflict_stream_array_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab3` +-- conflict_stream_array from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_array +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql new file mode 100644 index 000000000000..2a7669945831 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_name` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + id, + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab3` +-- conflict_stream_name from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_name +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql new file mode 100644 index 000000000000..1d897d05e549 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_name_conflict_stream_name` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_conflict_stream_name_hashid, + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_2_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab3` +-- conflict_stream_name at conflict_stream_name/conflict_stream_name from `dataline-integration-testing`.test_normalization.`conflict_stream_name` +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql new file mode 100644 index 000000000000..033f337977b9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_conflict_stream_name_2_hashid, + `groups`, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_3_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3` +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from `dataline-integration-testing`.test_normalization.`conflict_stream_name_conflict_stream_name` +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql new file mode 100644 index 000000000000..b5b99cd91982 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_scalar` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + id, + conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_conflict_stream_scalar_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab3` +-- conflict_stream_scalar from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_scalar +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql new file mode 100644 index 000000000000..9e5aca960cfd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql @@ -0,0 +1,23 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + double_array_data, + DATA, + column___with__quotes, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_partition_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3` +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names` +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql new file mode 100644 index 000000000000..d8f8943f5149 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_DATA_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3` +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql new file mode 100644 index 000000000000..01a20f04da09 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_column___with__quotes_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3` +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql new file mode 100644 index 000000000000..639fa690b35b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_partition_hashid, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_double_array_data_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3` +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql new file mode 100644 index 000000000000..880063feec3c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + id, + date, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab3` +-- non_nested_stream_without_namespace_resulting_into_long_names from `dataline-integration-testing`.test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql new file mode 100644 index 000000000000..b0aa1b41329f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -0,0 +1,22 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`unnest_alias_children` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_unnest_alias_hashid, + ab_id, + owner, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_children_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab3` +-- children at unnest_alias/children from `dataline-integration-testing`.test_normalization.`unnest_alias` +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql new file mode 100644 index 000000000000..4be0121256bd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`unnest_alias_children_owner` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_children_hashid, + owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_owner_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab3` +-- owner at unnest_alias/children/owner from `dataline-integration-testing`.test_normalization.`unnest_alias_children` +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql new file mode 100644 index 000000000000..b2236adf2322 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + id, + date, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid +from `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab3` +-- simple_stream_with_namespace_resulting_into_long_names from `dataline-integration-testing`.test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql new file mode 100644 index 000000000000..36e9afe3d413 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql @@ -0,0 +1,22 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(date as + string +), ''), '-', coalesce(cast(`partition` as + string +), '')) as + string +))) as _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab2` tmp +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql deleted file mode 100644 index ad44e5406be9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_scalar(_airbyte_data, "$['name']") as name, - json_extract_scalar(_airbyte_data, "$['column___with__quotes']") as column___with__quotes, - json_extract_scalar(_airbyte_data, "$['_ab_cdc_lsn']") as _ab_cdc_lsn, - json_extract_scalar(_airbyte_data, "$['_ab_cdc_updated_at']") as _ab_cdc_updated_at, - json_extract_scalar(_airbyte_data, "$['_ab_cdc_deleted_at']") as _ab_cdc_deleted_at, - _airbyte_emitted_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias --- dedup_cdc_excluded; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql deleted file mode 100644 index 623a8fc37b01..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,29 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - int64 -) as id, - cast(name as - string -) as name, - cast(column___with__quotes as - string -) as column___with__quotes, - cast(_ab_cdc_lsn as - float64 -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float64 -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float64 -) as _ab_cdc_deleted_at, - _airbyte_emitted_at -from `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab1` --- dedup_cdc_excluded; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index e5edf0279062..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,26 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(name as - string -), ''), '-', coalesce(cast(column___with__quotes as - string -), ''), '-', coalesce(cast(_ab_cdc_lsn as - string -), ''), '-', coalesce(cast(_ab_cdc_updated_at as - string -), ''), '-', coalesce(cast(_ab_cdc_deleted_at as - string -), '')) as - string -))) as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab2` tmp --- dedup_cdc_excluded; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql deleted file mode 100644 index a6e50ca2981a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab4` - OPTIONS() - as --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_cdc_excluded_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab3` tmp --- dedup_cdc_excluded from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_cdc_excluded; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql deleted file mode 100644 index 75554c2c95f6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_scalar(_airbyte_data, "$['currency']") as currency, - json_extract_scalar(_airbyte_data, "$['date']") as date, - json_extract_scalar(_airbyte_data, "$['timestamp_col']") as timestamp_col, - json_extract_scalar(_airbyte_data, "$['HKD@spéçiäl & characters']") as HKD_special___characters, - json_extract_scalar(_airbyte_data, "$['HKD_special___characters']") as HKD_special___characters_1, - json_extract_scalar(_airbyte_data, "$['NZD']") as NZD, - json_extract_scalar(_airbyte_data, "$['USD']") as USD, - _airbyte_emitted_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_exchange_rate as table_alias --- dedup_exchange_rate; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql deleted file mode 100644 index dc63d566b553..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ /dev/null @@ -1,35 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - int64 -) as id, - cast(currency as - string -) as currency, - cast(nullif(date, '') as - date -) as date, - cast(nullif(timestamp_col, '') as - timestamp -) as timestamp_col, - cast(HKD_special___characters as - float64 -) as HKD_special___characters, - cast(HKD_special___characters_1 as - string -) as HKD_special___characters_1, - cast(NZD as - float64 -) as NZD, - cast(USD as - float64 -) as USD, - _airbyte_emitted_at -from `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab1` --- dedup_exchange_rate; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql deleted file mode 100644 index 96c11756002a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql +++ /dev/null @@ -1,30 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(currency as - string -), ''), '-', coalesce(cast(date as - string -), ''), '-', coalesce(cast(timestamp_col as - string -), ''), '-', coalesce(cast(HKD_special___characters as - string -), ''), '-', coalesce(cast(HKD_special___characters_1 as - string -), ''), '-', coalesce(cast(NZD as - string -), ''), '-', coalesce(cast(USD as - string -), '')) as - string -))) as _airbyte_dedup_exchange_rate_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab2` tmp --- dedup_exchange_rate; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql deleted file mode 100644 index 683c9be94e24..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab4` - OPTIONS() - as --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_exchange_rate_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab3` tmp --- dedup_exchange_rate from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_exchange_rate; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab1.sql deleted file mode 100644 index a7ed234497d7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_scalar(_airbyte_data, "$['currency']") as currency, - json_extract_scalar(_airbyte_data, "$['date']") as date, - json_extract_scalar(_airbyte_data, "$['timestamp_col']") as timestamp_col, - json_extract_scalar(_airbyte_data, "$['HKD@spéçiäl & characters']") as HKD_special___characters, - json_extract_scalar(_airbyte_data, "$['HKD_special___characters']") as HKD_special___characters_1, - json_extract_scalar(_airbyte_data, "$['NZD']") as NZD, - json_extract_scalar(_airbyte_data, "$['USD']") as USD, - _airbyte_emitted_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_exchange_rate as table_alias --- exchange_rate; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab2.sql deleted file mode 100644 index b5299bf030b2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ /dev/null @@ -1,35 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - int64 -) as id, - cast(currency as - string -) as currency, - cast(nullif(date, '') as - date -) as date, - cast(nullif(timestamp_col, '') as - timestamp -) as timestamp_col, - cast(HKD_special___characters as - float64 -) as HKD_special___characters, - cast(HKD_special___characters_1 as - string -) as HKD_special___characters_1, - cast(NZD as - float64 -) as NZD, - cast(USD as - float64 -) as USD, - _airbyte_emitted_at -from `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab1` --- exchange_rate; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab3.sql deleted file mode 100644 index cefe4b9e1dfc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ /dev/null @@ -1,30 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(currency as - string -), ''), '-', coalesce(cast(date as - string -), ''), '-', coalesce(cast(timestamp_col as - string -), ''), '-', coalesce(cast(HKD_special___characters as - string -), ''), '-', coalesce(cast(HKD_special___characters_1 as - string -), ''), '-', coalesce(cast(NZD as - string -), ''), '-', coalesce(cast(USD as - string -), '')) as - string -))) as _airbyte_exchange_rate_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab2` tmp --- exchange_rate; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql deleted file mode 100644 index 90058783d998..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_scalar(_airbyte_data, "$['name']") as name, - json_extract_scalar(_airbyte_data, "$['_ab_cdc_lsn']") as _ab_cdc_lsn, - json_extract_scalar(_airbyte_data, "$['_ab_cdc_updated_at']") as _ab_cdc_updated_at, - json_extract_scalar(_airbyte_data, "$['_ab_cdc_deleted_at']") as _ab_cdc_deleted_at, - json_extract_scalar(_airbyte_data, "$['_ab_cdc_log_pos']") as _ab_cdc_log_pos, - _airbyte_emitted_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias --- pos_dedup_cdcx; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql deleted file mode 100644 index 176afed37e9f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ /dev/null @@ -1,29 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - int64 -) as id, - cast(name as - string -) as name, - cast(_ab_cdc_lsn as - float64 -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float64 -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float64 -) as _ab_cdc_deleted_at, - cast(_ab_cdc_log_pos as - float64 -) as _ab_cdc_log_pos, - _airbyte_emitted_at -from `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab1` --- pos_dedup_cdcx; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index 3cdea4f9f809..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,26 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(name as - string -), ''), '-', coalesce(cast(_ab_cdc_lsn as - string -), ''), '-', coalesce(cast(_ab_cdc_updated_at as - string -), ''), '-', coalesce(cast(_ab_cdc_deleted_at as - string -), ''), '-', coalesce(cast(_ab_cdc_log_pos as - string -), '')) as - string -))) as _airbyte_pos_dedup_cdcx_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab2` tmp --- pos_dedup_cdcx; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql deleted file mode 100644 index 94c22e794dd8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab4` - OPTIONS() - as --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_pos_dedup_cdcx_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab3` tmp --- pos_dedup_cdcx from `dataline-integration-testing`.test_normalization._airbyte_raw_pos_dedup_cdcx; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index 248956c01dad..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,23 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`dedup_cdc_excluded` - - - OPTIONS() - as ( - --- Final base SQL model -select - id, - name, - column___with__quotes, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from `dataline-integration-testing`.test_normalization.`dedup_cdc_excluded_scd` --- dedup_cdc_excluded from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_cdc_excluded -where _airbyte_active_row = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index 307d80f772dc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,32 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`dedup_cdc_excluded_scd` - - - OPTIONS() - as ( - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - column___with__quotes, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab4` --- dedup_cdc_excluded from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_cdc_excluded -where _airbyte_row_num = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql deleted file mode 100644 index 32e1e0d363fa..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql +++ /dev/null @@ -1,25 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`dedup_exchange_rate` - - - OPTIONS() - as ( - --- Final base SQL model -select - id, - currency, - date, - timestamp_col, - HKD_special___characters, - HKD_special___characters_1, - NZD, - USD, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from `dataline-integration-testing`.test_normalization.`dedup_exchange_rate_scd` --- dedup_exchange_rate from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_exchange_rate -where _airbyte_active_row = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql deleted file mode 100644 index d164be902d73..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql +++ /dev/null @@ -1,38 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`dedup_exchange_rate_scd` - - - OPTIONS() - as ( - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - currency, - date, - timestamp_col, - HKD_special___characters, - HKD_special___characters_1, - NZD, - USD, - date as _airbyte_start_at, - lag(date) over ( - partition by id, currency, cast(NZD as - string -) - order by date is null asc, date desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(date) over ( - partition by id, currency, cast(NZD as - string -) - order by date is null asc, date desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab4` --- dedup_exchange_rate from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_exchange_rate -where _airbyte_row_num = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql deleted file mode 100644 index 364bdfec888d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,24 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`exchange_rate` - - - OPTIONS() - as ( - --- Final base SQL model -select - id, - currency, - date, - timestamp_col, - HKD_special___characters, - HKD_special___characters_1, - NZD, - USD, - _airbyte_emitted_at, - _airbyte_exchange_rate_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab3` --- exchange_rate from `dataline-integration-testing`.test_normalization._airbyte_raw_exchange_rate - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql deleted file mode 100644 index 62ca5f80f979..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ /dev/null @@ -1,23 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`pos_dedup_cdcx` - - - OPTIONS() - as ( - --- Final base SQL model -select - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid -from `dataline-integration-testing`.test_normalization.`pos_dedup_cdcx_scd` --- pos_dedup_cdcx from `dataline-integration-testing`.test_normalization._airbyte_raw_pos_dedup_cdcx -where _airbyte_active_row = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index 12a8c74c9c08..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,32 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`pos_dedup_cdcx_scd` - - - OPTIONS() - as ( - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab4` --- pos_dedup_cdcx from `dataline-integration-testing`.test_normalization._airbyte_raw_pos_dedup_cdcx -where _airbyte_row_num = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..eecd7c33da6d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -0,0 +1,20 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['name']") as name, + json_extract_scalar(_airbyte_data, "$['_ab_cdc_lsn']") as _ab_cdc_lsn, + json_extract_scalar(_airbyte_data, "$['_ab_cdc_updated_at']") as _ab_cdc_updated_at, + json_extract_scalar(_airbyte_data, "$['_ab_cdc_deleted_at']") as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias +-- dedup_cdc_excluded +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..995249ea89b1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -0,0 +1,30 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + int64 +) as id, + cast(name as + string +) as name, + cast(_ab_cdc_lsn as + float64 +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float64 +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float64 +) as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab1` +-- dedup_cdc_excluded +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql new file mode 100644 index 000000000000..aba44bffd591 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -0,0 +1,23 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['currency']") as currency, + json_extract_scalar(_airbyte_data, "$['date']") as date, + json_extract_scalar(_airbyte_data, "$['timestamp_col']") as timestamp_col, + json_extract_scalar(_airbyte_data, "$['HKD@spéçiäl & characters']") as HKD_special___characters, + json_extract_scalar(_airbyte_data, "$['HKD_special___characters']") as HKD_special___characters_1, + json_extract_scalar(_airbyte_data, "$['NZD']") as NZD, + json_extract_scalar(_airbyte_data, "$['USD']") as USD, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql new file mode 100644 index 000000000000..37dc10cde3ee --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -0,0 +1,39 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + int64 +) as id, + cast(currency as + string +) as currency, + cast(nullif(date, '') as + date +) as date, + cast(nullif(timestamp_col, '') as + timestamp +) as timestamp_col, + cast(HKD_special___characters as + float64 +) as HKD_special___characters, + cast(HKD_special___characters_1 as + string +) as HKD_special___characters_1, + cast(NZD as + float64 +) as NZD, + cast(USD as + float64 +) as USD, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab1` +-- dedup_exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql new file mode 100644 index 000000000000..7e55806a2d74 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -0,0 +1,23 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['currency']") as currency, + json_extract_scalar(_airbyte_data, "$['date']") as date, + json_extract_scalar(_airbyte_data, "$['timestamp_col']") as timestamp_col, + json_extract_scalar(_airbyte_data, "$['HKD@spéçiäl & characters']") as HKD_special___characters, + json_extract_scalar(_airbyte_data, "$['HKD_special___characters']") as HKD_special___characters_1, + json_extract_scalar(_airbyte_data, "$['NZD']") as NZD, + json_extract_scalar(_airbyte_data, "$['USD']") as USD, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql new file mode 100644 index 000000000000..2bfffd7d4b36 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -0,0 +1,39 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + int64 +) as id, + cast(currency as + string +) as currency, + cast(nullif(date, '') as + date +) as date, + cast(nullif(timestamp_col, '') as + timestamp +) as timestamp_col, + cast(HKD_special___characters as + float64 +) as HKD_special___characters, + cast(HKD_special___characters_1 as + string +) as HKD_special___characters_1, + cast(NZD as + float64 +) as NZD, + cast(USD as + float64 +) as USD, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab1` +-- exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql new file mode 100644 index 000000000000..4c66cad39bd6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -0,0 +1,32 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(currency as + string +), ''), '-', coalesce(cast(date as + string +), ''), '-', coalesce(cast(timestamp_col as + string +), ''), '-', coalesce(cast(HKD_special___characters as + string +), ''), '-', coalesce(cast(HKD_special___characters_1 as + string +), ''), '-', coalesce(cast(NZD as + string +), ''), '-', coalesce(cast(USD as + string +), '')) as + string +))) as _airbyte_exchange_rate_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab2` tmp +-- exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql new file mode 100644 index 000000000000..267f6e648459 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -0,0 +1,20 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['name']") as name, + json_extract_scalar(_airbyte_data, "$['_ab_cdc_lsn']") as _ab_cdc_lsn, + json_extract_scalar(_airbyte_data, "$['_ab_cdc_updated_at']") as _ab_cdc_updated_at, + json_extract_scalar(_airbyte_data, "$['_ab_cdc_deleted_at']") as _ab_cdc_deleted_at, + json_extract_scalar(_airbyte_data, "$['_ab_cdc_log_pos']") as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias +-- pos_dedup_cdcx +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql new file mode 100644 index 000000000000..58262d8eaaa5 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -0,0 +1,32 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + int64 +) as id, + cast(name as + string +) as name, + cast(_ab_cdc_lsn as + float64 +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float64 +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float64 +) as _ab_cdc_deleted_at, + cast(_ab_cdc_log_pos as + float64 +) as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab1` +-- pos_dedup_cdcx +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..9c881dde8e67 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,16 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..b427d44957c8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + int64 +) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1` +-- renamed_dedup_cdc_excluded +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..4b61673a2f50 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,98 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`dedup_cdc_excluded_scd` + partition by range_bucket( + _airbyte_active_row, + generate_array(0, 1, 1) + ) + cluster by _airbyte_unique_key, _airbyte_emitted_at + OPTIONS() + as ( + +with + +input_data as ( + select * + from `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab3` + -- dedup_cdc_excluded from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), '')) as + string +))) as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as + string +), cast(_ab_cdc_updated_at as + string +) + order by _airbyte_ab_id + ) as _airbyte_row_num, + to_hex(md5(cast(concat(coalesce(cast(_airbyte_unique_key as + string +), ''), '-', coalesce(cast(_airbyte_start_at as + string +), ''), '-', coalesce(cast(_airbyte_emitted_at as + string +), ''), '-', coalesce(cast(_ab_cdc_deleted_at as + string +), ''), '-', coalesce(cast(_ab_cdc_updated_at as + string +), '')) as + string +))) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..12494cc91c84 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,82 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded_scd` + partition by range_bucket( + _airbyte_active_row, + generate_array(0, 1, 1) + ) + cluster by _airbyte_unique_key, _airbyte_emitted_at + OPTIONS() + as ( + +with + +input_data as ( + select * + from `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab3` + -- renamed_dedup_cdc_excluded from `dataline-integration-testing`.test_normalization._airbyte_raw_renamed_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), '')) as + string +))) as _airbyte_unique_key, + id, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_renamed_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + to_hex(md5(cast(concat(coalesce(cast(_airbyte_unique_key as + string +), ''), '-', coalesce(cast(_airbyte_start_at as + string +), ''), '-', coalesce(cast(_airbyte_emitted_at as + string +), '')) as + string +))) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql new file mode 100644 index 000000000000..79eebde9c581 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -0,0 +1,27 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`dedup_cdc_excluded` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_unique_key, _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from `dataline-integration-testing`.test_normalization.`dedup_cdc_excluded_scd` +-- dedup_cdc_excluded from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_cdc_excluded +where 1 = 1 +and _airbyte_active_row = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..fcfbe454ca5d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,30 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`dedup_exchange_rate` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_unique_key, _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + currency, + date, + timestamp_col, + HKD_special___characters, + HKD_special___characters_1, + NZD, + USD, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from `dataline-integration-testing`.test_normalization.`dedup_exchange_rate_scd` +-- dedup_exchange_rate from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_exchange_rate +where 1 = 1 +and _airbyte_active_row = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..c49e28422cd3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -0,0 +1,28 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`exchange_rate` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + id, + currency, + date, + timestamp_col, + HKD_special___characters, + HKD_special___characters_1, + NZD, + USD, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab3` +-- exchange_rate from `dataline-integration-testing`.test_normalization._airbyte_raw_exchange_rate +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..1b67b9d0c597 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,23 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_unique_key, _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded_scd` +-- renamed_dedup_cdc_excluded from `dataline-integration-testing`.test_normalization._airbyte_raw_renamed_dedup_cdc_excluded +where 1 = 1 +and _airbyte_active_row = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..20c879115c65 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,104 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`pos_dedup_cdcx_scd` + partition by range_bucket( + _airbyte_active_row, + generate_array(0, 1, 1) + ) + cluster by _airbyte_unique_key, _airbyte_emitted_at + OPTIONS() + as ( + +with + +input_data as ( + select * + from `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab3` + -- pos_dedup_cdcx from `dataline-integration-testing`.test_normalization._airbyte_raw_pos_dedup_cdcx +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), '')) as + string +))) as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as + string +), cast(_ab_cdc_updated_at as + string +), cast(_ab_cdc_log_pos as + string +) + order by _airbyte_ab_id + ) as _airbyte_row_num, + to_hex(md5(cast(concat(coalesce(cast(_airbyte_unique_key as + string +), ''), '-', coalesce(cast(_airbyte_start_at as + string +), ''), '-', coalesce(cast(_airbyte_emitted_at as + string +), ''), '-', coalesce(cast(_ab_cdc_deleted_at as + string +), ''), '-', coalesce(cast(_ab_cdc_updated_at as + string +), ''), '-', coalesce(cast(_ab_cdc_log_pos as + string +), '')) as + string +))) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql new file mode 100644 index 000000000000..bde8bc2db495 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -0,0 +1,27 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`pos_dedup_cdcx` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_unique_key, _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from `dataline-integration-testing`.test_normalization.`pos_dedup_cdcx_scd` +-- pos_dedup_cdcx from `dataline-integration-testing`.test_normalization._airbyte_raw_pos_dedup_cdcx +where 1 = 1 +and _airbyte_active_row = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..662067add2e0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,26 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(name as + string +), ''), '-', coalesce(cast(_ab_cdc_lsn as + string +), ''), '-', coalesce(cast(_ab_cdc_updated_at as + string +), ''), '-', coalesce(cast(_ab_cdc_deleted_at as + string +), '')) as + string +))) as _airbyte_dedup_cdc_excluded_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab2` tmp +-- dedup_cdc_excluded +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..4ed21e656fe2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,32 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(currency as + string +), ''), '-', coalesce(cast(date as + string +), ''), '-', coalesce(cast(timestamp_col as + string +), ''), '-', coalesce(cast(HKD_special___characters as + string +), ''), '-', coalesce(cast(HKD_special___characters_1 as + string +), ''), '-', coalesce(cast(NZD as + string +), ''), '-', coalesce(cast(USD as + string +), '')) as + string +))) as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab2` tmp +-- dedup_exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql new file mode 100644 index 000000000000..4936ef6153fe --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -0,0 +1,27 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(name as + string +), ''), '-', coalesce(cast(_ab_cdc_lsn as + string +), ''), '-', coalesce(cast(_ab_cdc_updated_at as + string +), ''), '-', coalesce(cast(_ab_cdc_deleted_at as + string +), ''), '-', coalesce(cast(_ab_cdc_log_pos as + string +), '')) as + string +))) as _airbyte_pos_dedup_cdcx_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab2` tmp +-- pos_dedup_cdcx +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..016b91d7bded --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), '')) as + string +))) as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2` tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql deleted file mode 100644 index 6144a13617e8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql +++ /dev/null @@ -1,43 +0,0 @@ -with table_row_counts as ( - select distinct count(*) as row_count, 10 as expected_count - from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} -union all - select distinct count(*) as row_count, 10 as expected_count - from {{ ref('exchange_rate') }} - -union all - - select distinct count(*) as row_count, 10 as expected_count - from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -union all - select distinct count(*) as row_count, 10 as expected_count - from {{ ref('dedup_exchange_rate_scd') }} -union all - select distinct count(*) as row_count, 5 as expected_count - from {{ ref('dedup_exchange_rate') }} - -union all - - select distinct count(*) as row_count, 8 as expected_count - from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -union all - select distinct count(*) as row_count, 8 as expected_count - from {{ ref('dedup_cdc_excluded_scd') }} -union all - select distinct count(*) as row_count, 4 as expected_count - from {{ ref('dedup_cdc_excluded') }} - -union all - - select distinct count(*) as row_count, 8 as expected_count - from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -union all - select distinct count(*) as row_count, 8 as expected_count - from {{ ref('pos_dedup_cdcx_scd') }} -union all - select distinct count(*) as row_count, 3 as expected_count - from {{ ref('pos_dedup_cdcx') }} -) -select * -from table_row_counts -where row_count != expected_count diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql index f1e32ebde640..fa3068bcaa9f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -1,13 +1,22 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['name'], ['name']) }} as name, - {{ json_extract_scalar('_airbyte_data', ['column`_\'with"_quotes'], ['column___with__quotes']) }} as column___with__quotes, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_lsn'], ['_ab_cdc_lsn']) }} as _ab_cdc_lsn, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} as table_alias -- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql index 2b50d6d18d95..322f1de599d7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -1,13 +1,22 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, cast(name as {{ dbt_utils.type_string() }}) as name, - cast(column___with__quotes as {{ dbt_utils.type_string() }}) as column___with__quotes, cast(_ab_cdc_lsn as {{ dbt_utils.type_float() }}) as _ab_cdc_lsn, cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('dedup_cdc_excluded_ab1') }} -- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 6fa998e293a0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'name', - 'column___with__quotes', - '_ab_cdc_lsn', - '_ab_cdc_updated_at', - '_ab_cdc_deleted_at', - ]) }} as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from {{ ref('dedup_cdc_excluded_ab2') }} tmp --- dedup_cdc_excluded - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql deleted file mode 100644 index aa34179c3c5c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_cdc_excluded_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('dedup_cdc_excluded_ab3') }} tmp --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql index 2e5c76354fb8..a1925d6ed311 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, @@ -9,7 +15,11 @@ select {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as HKD_special___characters_1, {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as NZD, {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as USD, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} as table_alias -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index 4604bd5acb63..3790a0629706 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -9,7 +15,11 @@ select cast(HKD_special___characters_1 as {{ dbt_utils.type_string() }}) as HKD_special___characters_1, cast(NZD as {{ dbt_utils.type_float() }}) as NZD, cast(USD as {{ dbt_utils.type_float() }}) as USD, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('dedup_exchange_rate_ab1') }} -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql deleted file mode 100644 index abd8582a7f1a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'currency', - 'date', - 'timestamp_col', - 'HKD_special___characters', - 'HKD_special___characters_1', - 'NZD', - 'USD', - ]) }} as _airbyte_dedup_exchange_rate_hashid, - tmp.* -from {{ ref('dedup_exchange_rate_ab2') }} tmp --- dedup_exchange_rate - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql deleted file mode 100644 index ef5f6e52d60e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_exchange_rate_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('dedup_exchange_rate_ab3') }} tmp --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql index 0cac51cb1e9c..51af17b7e5c1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, @@ -9,7 +15,11 @@ select {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as HKD_special___characters_1, {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as NZD, {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as USD, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} as table_alias -- exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql index 0182b4f09c20..319bc81f8575 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -9,7 +15,11 @@ select cast(HKD_special___characters_1 as {{ dbt_utils.type_string() }}) as HKD_special___characters_1, cast(NZD as {{ dbt_utils.type_float() }}) as NZD, cast(USD as {{ dbt_utils.type_float() }}) as USD, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('exchange_rate_ab1') }} -- exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql index 07c673d18e8e..0c6628f131eb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -14,4 +20,6 @@ select tmp.* from {{ ref('exchange_rate_ab2') }} tmp -- exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql index 1041493c6c67..b69171ca3358 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, @@ -7,7 +13,10 @@ select {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_log_pos'], ['_ab_cdc_log_pos']) }} as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} as table_alias -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql index bccb189a9d7a..40c6e7e9aa85 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -7,7 +13,10 @@ select cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, cast(_ab_cdc_log_pos as {{ dbt_utils.type_float() }}) as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('pos_dedup_cdcx_ab1') }} -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index 0fd3196cea9f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'name', - '_ab_cdc_lsn', - '_ab_cdc_updated_at', - '_ab_cdc_deleted_at', - '_ab_cdc_log_pos', - ]) }} as _airbyte_pos_dedup_cdcx_hashid, - tmp.* -from {{ ref('pos_dedup_cdcx_ab2') }} tmp --- pos_dedup_cdcx - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql deleted file mode 100644 index 7c897124b395..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_pos_dedup_cdcx_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('pos_dedup_cdcx_ab3') }} tmp --- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..221619f8f03c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,18 @@ +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..c9cd56d5d915 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,18 @@ +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as {{ dbt_utils.type_bigint() }}) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ ref('renamed_dedup_cdc_excluded_ab1') }} +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..69b881ca1ed9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,113 @@ +{{ config( + cluster_by = ["_airbyte_unique_key","_airbyte_emitted_at"], + partition_by = {"field": "_airbyte_active_row", "data_type": "int64", "range": {"start": 0, "end": 1, "interval": 1}}, + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('dedup_cdc_excluded_ab3') }} + -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('dedup_cdc_excluded_ab3') }} + -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}) + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 35175abb8ed5..aff096b2e6f1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -9,7 +9,7 @@ with {% if is_incremental() %} new_data as ( -- retrieve incremental "new" data - select + select * from {{ ref('dedup_exchange_rate_ab3') }} -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} @@ -18,7 +18,7 @@ new_data as ( ), new_data_ids as ( -- build a subset of _airbyte_unique_key from rows that are new - select distinct + select distinct {{ dbt_utils.surrogate_key([ 'id', 'currency', diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..f008ba4b8401 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,105 @@ +{{ config( + cluster_by = ["_airbyte_unique_key","_airbyte_emitted_at"], + partition_by = {"field": "_airbyte_active_row", "data_type": "int64", "range": {"start": 0, "end": 1, "interval": 1}}, + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('renamed_dedup_cdc_excluded_ab3') }} + -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('renamed_dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('renamed_dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('renamed_dedup_cdc_excluded_ab3') }} + -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_renamed_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql new file mode 100644 index 000000000000..14050eec8255 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -0,0 +1,25 @@ +{{ config( + cluster_by = ["_airbyte_unique_key","_airbyte_emitted_at"], + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from {{ ref('dedup_cdc_excluded_scd') }} +-- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..57d78ad43bf1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,28 @@ +{{ config( + cluster_by = ["_airbyte_unique_key","_airbyte_emitted_at"], + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + id, + currency, + date, + timestamp_col, + HKD_special___characters, + HKD_special___characters_1, + NZD, + USD, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from {{ ref('dedup_exchange_rate_scd') }} +-- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..c579db34bb15 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql @@ -0,0 +1,26 @@ +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + id, + currency, + date, + timestamp_col, + HKD_special___characters, + HKD_special___characters_1, + NZD, + USD, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from {{ ref('exchange_rate_ab3') }} +-- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..75212a4d2aa8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,21 @@ +{{ config( + cluster_by = ["_airbyte_unique_key","_airbyte_emitted_at"], + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from {{ ref('renamed_dedup_cdc_excluded_scd') }} +-- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..58db901dc577 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,115 @@ +{{ config( + cluster_by = ["_airbyte_unique_key","_airbyte_emitted_at"], + partition_by = {"field": "_airbyte_active_row", "data_type": "int64", "range": {"start": 0, "end": 1, "interval": 1}}, + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('pos_dedup_cdcx_ab3') }} + -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('pos_dedup_cdcx_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('pos_dedup_cdcx_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('pos_dedup_cdcx_ab3') }} + -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_log_pos as {{ dbt_utils.type_string() }}) + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at', '_ab_cdc_log_pos' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index 5b81f35bfd03..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - name, - column___with__quotes, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from {{ ref('dedup_cdc_excluded_scd') }} --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -where _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index 0f91009def56..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - column___with__quotes, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from {{ ref('dedup_cdc_excluded_ab4') }} --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql deleted file mode 100644 index 85c70cd20c28..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - currency, - date, - timestamp_col, - HKD_special___characters, - HKD_special___characters_1, - NZD, - USD, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from {{ ref('dedup_exchange_rate_scd') }} --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -where _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql deleted file mode 100644 index 6f76314f0525..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql +++ /dev/null @@ -1,26 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - currency, - date, - timestamp_col, - HKD_special___characters, - HKD_special___characters_1, - NZD, - USD, - date as _airbyte_start_at, - lag(date) over ( - partition by id, currency, cast(NZD as {{ dbt_utils.type_string() }}) - order by date is null asc, date desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(date) over ( - partition by id, currency, cast(NZD as {{ dbt_utils.type_string() }}) - order by date is null asc, date desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from {{ ref('dedup_exchange_rate_ab4') }} --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql deleted file mode 100644 index 49c0da07de8e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - currency, - date, - timestamp_col, - HKD_special___characters, - HKD_special___characters_1, - NZD, - USD, - _airbyte_emitted_at, - _airbyte_exchange_rate_hashid -from {{ ref('exchange_rate_ab3') }} --- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql index ed02a5d04940..fe75b24654a8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -1,15 +1,25 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + cluster_by = ["_airbyte_unique_key","_airbyte_emitted_at"], + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select + _airbyte_unique_key, id, name, _ab_cdc_lsn, _ab_cdc_updated_at, _ab_cdc_deleted_at, _ab_cdc_log_pos, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_pos_dedup_cdcx_hashid from {{ ref('pos_dedup_cdcx_scd') }} -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index 81216ec938ee..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid -from {{ ref('pos_dedup_cdcx_ab4') }} --- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..46ac10d781f8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,22 @@ +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + 'name', + '_ab_cdc_lsn', + '_ab_cdc_updated_at', + '_ab_cdc_deleted_at', + ]) }} as _airbyte_dedup_cdc_excluded_hashid, + tmp.* +from {{ ref('dedup_cdc_excluded_ab2') }} tmp +-- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..2b0dd30ab3d7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,25 @@ +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + 'currency', + 'date', + 'timestamp_col', + 'HKD_special___characters', + 'HKD_special___characters_1', + 'NZD', + 'USD', + ]) }} as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from {{ ref('dedup_exchange_rate_ab2') }} tmp +-- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql new file mode 100644 index 000000000000..ee3dc14c336c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -0,0 +1,22 @@ +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + 'name', + '_ab_cdc_lsn', + '_ab_cdc_updated_at', + '_ab_cdc_deleted_at', + '_ab_cdc_log_pos', + ]) }} as _airbyte_pos_dedup_cdcx_hashid, + tmp.* +from {{ ref('pos_dedup_cdcx_ab2') }} tmp +-- pos_dedup_cdcx +where 1 = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..905b8f4c4087 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,18 @@ +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from {{ ref('renamed_dedup_cdc_excluded_ab2') }} tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/sources.yml index c085ca8ccc25..45c338b893ca 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/sources.yml @@ -10,3 +10,4 @@ sources: - name: _airbyte_raw_dedup_exchange_rate - name: _airbyte_raw_exchange_rate - name: _airbyte_raw_pos_dedup_cdcx + - name: _airbyte_raw_renamed_dedup_cdc_excluded diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..eecd7c33da6d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -0,0 +1,20 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['name']") as name, + json_extract_scalar(_airbyte_data, "$['_ab_cdc_lsn']") as _ab_cdc_lsn, + json_extract_scalar(_airbyte_data, "$['_ab_cdc_updated_at']") as _ab_cdc_updated_at, + json_extract_scalar(_airbyte_data, "$['_ab_cdc_deleted_at']") as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias +-- dedup_cdc_excluded +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..995249ea89b1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -0,0 +1,30 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + int64 +) as id, + cast(name as + string +) as name, + cast(_ab_cdc_lsn as + float64 +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float64 +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float64 +) as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab1` +-- dedup_cdc_excluded +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql new file mode 100644 index 000000000000..aba44bffd591 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -0,0 +1,23 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['currency']") as currency, + json_extract_scalar(_airbyte_data, "$['date']") as date, + json_extract_scalar(_airbyte_data, "$['timestamp_col']") as timestamp_col, + json_extract_scalar(_airbyte_data, "$['HKD@spéçiäl & characters']") as HKD_special___characters, + json_extract_scalar(_airbyte_data, "$['HKD_special___characters']") as HKD_special___characters_1, + json_extract_scalar(_airbyte_data, "$['NZD']") as NZD, + json_extract_scalar(_airbyte_data, "$['USD']") as USD, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql new file mode 100644 index 000000000000..37dc10cde3ee --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -0,0 +1,39 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + int64 +) as id, + cast(currency as + string +) as currency, + cast(nullif(date, '') as + date +) as date, + cast(nullif(timestamp_col, '') as + timestamp +) as timestamp_col, + cast(HKD_special___characters as + float64 +) as HKD_special___characters, + cast(HKD_special___characters_1 as + string +) as HKD_special___characters_1, + cast(NZD as + float64 +) as NZD, + cast(USD as + float64 +) as USD, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab1` +-- dedup_exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql new file mode 100644 index 000000000000..7e55806a2d74 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -0,0 +1,23 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['currency']") as currency, + json_extract_scalar(_airbyte_data, "$['date']") as date, + json_extract_scalar(_airbyte_data, "$['timestamp_col']") as timestamp_col, + json_extract_scalar(_airbyte_data, "$['HKD@spéçiäl & characters']") as HKD_special___characters, + json_extract_scalar(_airbyte_data, "$['HKD_special___characters']") as HKD_special___characters_1, + json_extract_scalar(_airbyte_data, "$['NZD']") as NZD, + json_extract_scalar(_airbyte_data, "$['USD']") as USD, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql new file mode 100644 index 000000000000..2bfffd7d4b36 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -0,0 +1,39 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + int64 +) as id, + cast(currency as + string +) as currency, + cast(nullif(date, '') as + date +) as date, + cast(nullif(timestamp_col, '') as + timestamp +) as timestamp_col, + cast(HKD_special___characters as + float64 +) as HKD_special___characters, + cast(HKD_special___characters_1 as + string +) as HKD_special___characters_1, + cast(NZD as + float64 +) as NZD, + cast(USD as + float64 +) as USD, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab1` +-- exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql new file mode 100644 index 000000000000..4c66cad39bd6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -0,0 +1,32 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(currency as + string +), ''), '-', coalesce(cast(date as + string +), ''), '-', coalesce(cast(timestamp_col as + string +), ''), '-', coalesce(cast(HKD_special___characters as + string +), ''), '-', coalesce(cast(HKD_special___characters_1 as + string +), ''), '-', coalesce(cast(NZD as + string +), ''), '-', coalesce(cast(USD as + string +), '')) as + string +))) as _airbyte_exchange_rate_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab2` tmp +-- exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql new file mode 100644 index 000000000000..267f6e648459 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -0,0 +1,20 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['name']") as name, + json_extract_scalar(_airbyte_data, "$['_ab_cdc_lsn']") as _ab_cdc_lsn, + json_extract_scalar(_airbyte_data, "$['_ab_cdc_updated_at']") as _ab_cdc_updated_at, + json_extract_scalar(_airbyte_data, "$['_ab_cdc_deleted_at']") as _ab_cdc_deleted_at, + json_extract_scalar(_airbyte_data, "$['_ab_cdc_log_pos']") as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias +-- pos_dedup_cdcx +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql new file mode 100644 index 000000000000..58262d8eaaa5 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -0,0 +1,32 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + int64 +) as id, + cast(name as + string +) as name, + cast(_ab_cdc_lsn as + float64 +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float64 +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float64 +) as _ab_cdc_deleted_at, + cast(_ab_cdc_log_pos as + float64 +) as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab1` +-- pos_dedup_cdcx +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..9c881dde8e67 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,16 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..b427d44957c8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + int64 +) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1` +-- renamed_dedup_cdc_excluded +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..bef8e97a0a47 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,27 @@ + + + + + + + + merge into `dataline-integration-testing`.test_normalization.`dedup_cdc_excluded_scd` as DBT_INTERNAL_DEST + using ( + select * from `dataline-integration-testing`.test_normalization.`dedup_cdc_excluded_scd__dbt_tmp` + ) as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._airbyte_unique_key_scd = DBT_INTERNAL_DEST._airbyte_unique_key_scd + + + + when matched then update set + `_airbyte_unique_key` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key`,`_airbyte_unique_key_scd` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key_scd`,`id` = DBT_INTERNAL_SOURCE.`id`,`name` = DBT_INTERNAL_SOURCE.`name`,`_ab_cdc_lsn` = DBT_INTERNAL_SOURCE.`_ab_cdc_lsn`,`_ab_cdc_updated_at` = DBT_INTERNAL_SOURCE.`_ab_cdc_updated_at`,`_ab_cdc_deleted_at` = DBT_INTERNAL_SOURCE.`_ab_cdc_deleted_at`,`_airbyte_start_at` = DBT_INTERNAL_SOURCE.`_airbyte_start_at`,`_airbyte_end_at` = DBT_INTERNAL_SOURCE.`_airbyte_end_at`,`_airbyte_active_row` = DBT_INTERNAL_SOURCE.`_airbyte_active_row`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_dedup_cdc_excluded_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_dedup_cdc_excluded_hashid` + + + when not matched then insert + (`_airbyte_unique_key`, `_airbyte_unique_key_scd`, `id`, `name`, `_ab_cdc_lsn`, `_ab_cdc_updated_at`, `_ab_cdc_deleted_at`, `_airbyte_start_at`, `_airbyte_end_at`, `_airbyte_active_row`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_dedup_cdc_excluded_hashid`) + values + (`_airbyte_unique_key`, `_airbyte_unique_key_scd`, `id`, `name`, `_ab_cdc_lsn`, `_ab_cdc_updated_at`, `_ab_cdc_deleted_at`, `_airbyte_start_at`, `_airbyte_end_at`, `_airbyte_active_row`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_dedup_cdc_excluded_hashid`) + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..83688c333707 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,27 @@ + + + + + + + + merge into `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded_scd` as DBT_INTERNAL_DEST + using ( + select * from `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded_scd__dbt_tmp` + ) as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._airbyte_unique_key_scd = DBT_INTERNAL_DEST._airbyte_unique_key_scd + + + + when matched then update set + `_airbyte_unique_key` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key`,`_airbyte_unique_key_scd` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key_scd`,`id` = DBT_INTERNAL_SOURCE.`id`,`_airbyte_start_at` = DBT_INTERNAL_SOURCE.`_airbyte_start_at`,`_airbyte_end_at` = DBT_INTERNAL_SOURCE.`_airbyte_end_at`,`_airbyte_active_row` = DBT_INTERNAL_SOURCE.`_airbyte_active_row`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_renamed_dedup_cdc_excluded_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_renamed_dedup_cdc_excluded_hashid` + + + when not matched then insert + (`_airbyte_unique_key`, `_airbyte_unique_key_scd`, `id`, `_airbyte_start_at`, `_airbyte_end_at`, `_airbyte_active_row`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_renamed_dedup_cdc_excluded_hashid`) + values + (`_airbyte_unique_key`, `_airbyte_unique_key_scd`, `id`, `_airbyte_start_at`, `_airbyte_end_at`, `_airbyte_active_row`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_renamed_dedup_cdc_excluded_hashid`) + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql new file mode 100644 index 000000000000..5ad7a0737609 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -0,0 +1,27 @@ + + + + + + + + merge into `dataline-integration-testing`.test_normalization.`dedup_cdc_excluded` as DBT_INTERNAL_DEST + using ( + select * from `dataline-integration-testing`.test_normalization.`dedup_cdc_excluded__dbt_tmp` + ) as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._airbyte_unique_key = DBT_INTERNAL_DEST._airbyte_unique_key + + + + when matched then update set + `_airbyte_unique_key` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key`,`id` = DBT_INTERNAL_SOURCE.`id`,`name` = DBT_INTERNAL_SOURCE.`name`,`_ab_cdc_lsn` = DBT_INTERNAL_SOURCE.`_ab_cdc_lsn`,`_ab_cdc_updated_at` = DBT_INTERNAL_SOURCE.`_ab_cdc_updated_at`,`_ab_cdc_deleted_at` = DBT_INTERNAL_SOURCE.`_ab_cdc_deleted_at`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_dedup_cdc_excluded_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_dedup_cdc_excluded_hashid` + + + when not matched then insert + (`_airbyte_unique_key`, `id`, `name`, `_ab_cdc_lsn`, `_ab_cdc_updated_at`, `_ab_cdc_deleted_at`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_dedup_cdc_excluded_hashid`) + values + (`_airbyte_unique_key`, `id`, `name`, `_ab_cdc_lsn`, `_ab_cdc_updated_at`, `_ab_cdc_deleted_at`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_dedup_cdc_excluded_hashid`) + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..0691294c98c3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,27 @@ + + + + + + + + merge into `dataline-integration-testing`.test_normalization.`dedup_exchange_rate` as DBT_INTERNAL_DEST + using ( + select * from `dataline-integration-testing`.test_normalization.`dedup_exchange_rate__dbt_tmp` + ) as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._airbyte_unique_key = DBT_INTERNAL_DEST._airbyte_unique_key + + + + when matched then update set + `_airbyte_unique_key` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key`,`id` = DBT_INTERNAL_SOURCE.`id`,`currency` = DBT_INTERNAL_SOURCE.`currency`,`date` = DBT_INTERNAL_SOURCE.`date`,`timestamp_col` = DBT_INTERNAL_SOURCE.`timestamp_col`,`HKD_special___characters` = DBT_INTERNAL_SOURCE.`HKD_special___characters`,`HKD_special___characters_1` = DBT_INTERNAL_SOURCE.`HKD_special___characters_1`,`NZD` = DBT_INTERNAL_SOURCE.`NZD`,`USD` = DBT_INTERNAL_SOURCE.`USD`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_dedup_exchange_rate_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_dedup_exchange_rate_hashid` + + + when not matched then insert + (`_airbyte_unique_key`, `id`, `currency`, `date`, `timestamp_col`, `HKD_special___characters`, `HKD_special___characters_1`, `NZD`, `USD`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_dedup_exchange_rate_hashid`) + values + (`_airbyte_unique_key`, `id`, `currency`, `date`, `timestamp_col`, `HKD_special___characters`, `HKD_special___characters_1`, `NZD`, `USD`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_dedup_exchange_rate_hashid`) + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..cf06f296aea9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -0,0 +1,27 @@ + + + + + + + + merge into `dataline-integration-testing`.test_normalization.`exchange_rate` as DBT_INTERNAL_DEST + using ( + select * from `dataline-integration-testing`.test_normalization.`exchange_rate__dbt_tmp` + ) as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._airbyte_ab_id = DBT_INTERNAL_DEST._airbyte_ab_id + + + + when matched then update set + `id` = DBT_INTERNAL_SOURCE.`id`,`currency` = DBT_INTERNAL_SOURCE.`currency`,`date` = DBT_INTERNAL_SOURCE.`date`,`timestamp_col` = DBT_INTERNAL_SOURCE.`timestamp_col`,`HKD_special___characters` = DBT_INTERNAL_SOURCE.`HKD_special___characters`,`HKD_special___characters_1` = DBT_INTERNAL_SOURCE.`HKD_special___characters_1`,`NZD` = DBT_INTERNAL_SOURCE.`NZD`,`USD` = DBT_INTERNAL_SOURCE.`USD`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_exchange_rate_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_exchange_rate_hashid` + + + when not matched then insert + (`id`, `currency`, `date`, `timestamp_col`, `HKD_special___characters`, `HKD_special___characters_1`, `NZD`, `USD`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_exchange_rate_hashid`) + values + (`id`, `currency`, `date`, `timestamp_col`, `HKD_special___characters`, `HKD_special___characters_1`, `NZD`, `USD`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_exchange_rate_hashid`) + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..0a8394cf9abc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,27 @@ + + + + + + + + merge into `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded` as DBT_INTERNAL_DEST + using ( + select * from `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded__dbt_tmp` + ) as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._airbyte_unique_key = DBT_INTERNAL_DEST._airbyte_unique_key + + + + when matched then update set + `_airbyte_unique_key` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key`,`id` = DBT_INTERNAL_SOURCE.`id`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_renamed_dedup_cdc_excluded_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_renamed_dedup_cdc_excluded_hashid` + + + when not matched then insert + (`_airbyte_unique_key`, `id`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_renamed_dedup_cdc_excluded_hashid`) + values + (`_airbyte_unique_key`, `id`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_renamed_dedup_cdc_excluded_hashid`) + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..20c879115c65 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,104 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`pos_dedup_cdcx_scd` + partition by range_bucket( + _airbyte_active_row, + generate_array(0, 1, 1) + ) + cluster by _airbyte_unique_key, _airbyte_emitted_at + OPTIONS() + as ( + +with + +input_data as ( + select * + from `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab3` + -- pos_dedup_cdcx from `dataline-integration-testing`.test_normalization._airbyte_raw_pos_dedup_cdcx +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), '')) as + string +))) as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as + string +), cast(_ab_cdc_updated_at as + string +), cast(_ab_cdc_log_pos as + string +) + order by _airbyte_ab_id + ) as _airbyte_row_num, + to_hex(md5(cast(concat(coalesce(cast(_airbyte_unique_key as + string +), ''), '-', coalesce(cast(_airbyte_start_at as + string +), ''), '-', coalesce(cast(_airbyte_emitted_at as + string +), ''), '-', coalesce(cast(_ab_cdc_deleted_at as + string +), ''), '-', coalesce(cast(_ab_cdc_updated_at as + string +), ''), '-', coalesce(cast(_ab_cdc_log_pos as + string +), '')) as + string +))) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql new file mode 100644 index 000000000000..bde8bc2db495 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -0,0 +1,27 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`pos_dedup_cdcx` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_unique_key, _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from `dataline-integration-testing`.test_normalization.`pos_dedup_cdcx_scd` +-- pos_dedup_cdcx from `dataline-integration-testing`.test_normalization._airbyte_raw_pos_dedup_cdcx +where 1 = 1 +and _airbyte_active_row = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..662067add2e0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,26 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(name as + string +), ''), '-', coalesce(cast(_ab_cdc_lsn as + string +), ''), '-', coalesce(cast(_ab_cdc_updated_at as + string +), ''), '-', coalesce(cast(_ab_cdc_deleted_at as + string +), '')) as + string +))) as _airbyte_dedup_cdc_excluded_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab2` tmp +-- dedup_cdc_excluded +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..4ed21e656fe2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,32 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(currency as + string +), ''), '-', coalesce(cast(date as + string +), ''), '-', coalesce(cast(timestamp_col as + string +), ''), '-', coalesce(cast(HKD_special___characters as + string +), ''), '-', coalesce(cast(HKD_special___characters_1 as + string +), ''), '-', coalesce(cast(NZD as + string +), ''), '-', coalesce(cast(USD as + string +), '')) as + string +))) as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab2` tmp +-- dedup_exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql new file mode 100644 index 000000000000..4936ef6153fe --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -0,0 +1,27 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(name as + string +), ''), '-', coalesce(cast(_ab_cdc_lsn as + string +), ''), '-', coalesce(cast(_ab_cdc_updated_at as + string +), ''), '-', coalesce(cast(_ab_cdc_deleted_at as + string +), ''), '-', coalesce(cast(_ab_cdc_log_pos as + string +), '')) as + string +))) as _airbyte_pos_dedup_cdcx_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab2` tmp +-- pos_dedup_cdcx +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..016b91d7bded --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), '')) as + string +))) as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2` tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql new file mode 100644 index 000000000000..5efa724e88f2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -0,0 +1,23 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['currency']") as currency, + json_extract_scalar(_airbyte_data, "$['new_column']") as new_column, + json_extract_scalar(_airbyte_data, "$['date']") as date, + json_extract_scalar(_airbyte_data, "$['timestamp_col']") as timestamp_col, + json_extract_scalar(_airbyte_data, "$['HKD@spéçiäl & characters']") as HKD_special___characters, + json_extract_scalar(_airbyte_data, "$['NZD']") as NZD, + json_extract_scalar(_airbyte_data, "$['USD']") as USD, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql new file mode 100644 index 000000000000..45be772bd787 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -0,0 +1,39 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + float64 +) as id, + cast(currency as + string +) as currency, + cast(new_column as + float64 +) as new_column, + cast(nullif(date, '') as + date +) as date, + cast(nullif(timestamp_col, '') as + timestamp +) as timestamp_col, + cast(HKD_special___characters as + float64 +) as HKD_special___characters, + cast(NZD as + float64 +) as NZD, + cast(USD as + int64 +) as USD, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab1` +-- dedup_exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql new file mode 100644 index 000000000000..cceb45e4340f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -0,0 +1,23 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['currency']") as currency, + json_extract_scalar(_airbyte_data, "$['new_column']") as new_column, + json_extract_scalar(_airbyte_data, "$['date']") as date, + json_extract_scalar(_airbyte_data, "$['timestamp_col']") as timestamp_col, + json_extract_scalar(_airbyte_data, "$['HKD@spéçiäl & characters']") as HKD_special___characters, + json_extract_scalar(_airbyte_data, "$['NZD']") as NZD, + json_extract_scalar(_airbyte_data, "$['USD']") as USD, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql new file mode 100644 index 000000000000..840040e43543 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -0,0 +1,39 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + float64 +) as id, + cast(currency as + string +) as currency, + cast(new_column as + float64 +) as new_column, + cast(nullif(date, '') as + date +) as date, + cast(nullif(timestamp_col, '') as + timestamp +) as timestamp_col, + cast(HKD_special___characters as + float64 +) as HKD_special___characters, + cast(NZD as + float64 +) as NZD, + cast(USD as + float64 +) as USD, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab1` +-- exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql new file mode 100644 index 000000000000..79386be9f718 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -0,0 +1,32 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(currency as + string +), ''), '-', coalesce(cast(new_column as + string +), ''), '-', coalesce(cast(date as + string +), ''), '-', coalesce(cast(timestamp_col as + string +), ''), '-', coalesce(cast(HKD_special___characters as + string +), ''), '-', coalesce(cast(NZD as + string +), ''), '-', coalesce(cast(USD as + string +), '')) as + string +))) as _airbyte_exchange_rate_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab2` tmp +-- exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..8fc7d170fb05 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,20 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['name']") as name, + json_extract_scalar(_airbyte_data, "$['_ab_cdc_lsn']") as _ab_cdc_lsn, + json_extract_scalar(_airbyte_data, "$['_ab_cdc_updated_at']") as _ab_cdc_updated_at, + json_extract_scalar(_airbyte_data, "$['_ab_cdc_deleted_at']") as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..295be3450166 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,30 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + int64 +) as id, + cast(name as + string +) as name, + cast(_ab_cdc_lsn as + float64 +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float64 +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float64 +) as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1` +-- renamed_dedup_cdc_excluded +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..7b882d8b693c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,27 @@ + + + + + + + + merge into `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded_scd` as DBT_INTERNAL_DEST + using ( + select * from `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded_scd__dbt_tmp` + ) as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._airbyte_unique_key_scd = DBT_INTERNAL_DEST._airbyte_unique_key_scd + + + + when matched then update set + `_airbyte_unique_key` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key`,`_airbyte_unique_key_scd` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key_scd`,`id` = DBT_INTERNAL_SOURCE.`id`,`_airbyte_start_at` = DBT_INTERNAL_SOURCE.`_airbyte_start_at`,`_airbyte_end_at` = DBT_INTERNAL_SOURCE.`_airbyte_end_at`,`_airbyte_active_row` = DBT_INTERNAL_SOURCE.`_airbyte_active_row`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_renamed_dedup_cdc_excluded_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_renamed_dedup_cdc_excluded_hashid`,`name` = DBT_INTERNAL_SOURCE.`name`,`_ab_cdc_lsn` = DBT_INTERNAL_SOURCE.`_ab_cdc_lsn`,`_ab_cdc_updated_at` = DBT_INTERNAL_SOURCE.`_ab_cdc_updated_at`,`_ab_cdc_deleted_at` = DBT_INTERNAL_SOURCE.`_ab_cdc_deleted_at` + + + when not matched then insert + (`_airbyte_unique_key`, `_airbyte_unique_key_scd`, `id`, `_airbyte_start_at`, `_airbyte_end_at`, `_airbyte_active_row`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_renamed_dedup_cdc_excluded_hashid`, `name`, `_ab_cdc_lsn`, `_ab_cdc_updated_at`, `_ab_cdc_deleted_at`) + values + (`_airbyte_unique_key`, `_airbyte_unique_key_scd`, `id`, `_airbyte_start_at`, `_airbyte_end_at`, `_airbyte_active_row`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_renamed_dedup_cdc_excluded_hashid`, `name`, `_ab_cdc_lsn`, `_ab_cdc_updated_at`, `_ab_cdc_deleted_at`) + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..f5f7e5ca80df --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,27 @@ + + + + + + + + merge into `dataline-integration-testing`.test_normalization.`dedup_exchange_rate` as DBT_INTERNAL_DEST + using ( + select * from `dataline-integration-testing`.test_normalization.`dedup_exchange_rate__dbt_tmp` + ) as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._airbyte_unique_key = DBT_INTERNAL_DEST._airbyte_unique_key + + + + when matched then update set + `_airbyte_unique_key` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key`,`id` = DBT_INTERNAL_SOURCE.`id`,`currency` = DBT_INTERNAL_SOURCE.`currency`,`date` = DBT_INTERNAL_SOURCE.`date`,`timestamp_col` = DBT_INTERNAL_SOURCE.`timestamp_col`,`HKD_special___characters` = DBT_INTERNAL_SOURCE.`HKD_special___characters`,`NZD` = DBT_INTERNAL_SOURCE.`NZD`,`USD` = DBT_INTERNAL_SOURCE.`USD`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_dedup_exchange_rate_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_dedup_exchange_rate_hashid`,`new_column` = DBT_INTERNAL_SOURCE.`new_column` + + + when not matched then insert + (`_airbyte_unique_key`, `id`, `currency`, `date`, `timestamp_col`, `HKD_special___characters`, `NZD`, `USD`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_dedup_exchange_rate_hashid`, `new_column`) + values + (`_airbyte_unique_key`, `id`, `currency`, `date`, `timestamp_col`, `HKD_special___characters`, `NZD`, `USD`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_dedup_exchange_rate_hashid`, `new_column`) + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..b20cae81e4bf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -0,0 +1,27 @@ + + + + + + + + merge into `dataline-integration-testing`.test_normalization.`exchange_rate` as DBT_INTERNAL_DEST + using ( + select * from `dataline-integration-testing`.test_normalization.`exchange_rate__dbt_tmp` + ) as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._airbyte_ab_id = DBT_INTERNAL_DEST._airbyte_ab_id + + + + when matched then update set + `id` = DBT_INTERNAL_SOURCE.`id`,`currency` = DBT_INTERNAL_SOURCE.`currency`,`date` = DBT_INTERNAL_SOURCE.`date`,`timestamp_col` = DBT_INTERNAL_SOURCE.`timestamp_col`,`HKD_special___characters` = DBT_INTERNAL_SOURCE.`HKD_special___characters`,`NZD` = DBT_INTERNAL_SOURCE.`NZD`,`USD` = DBT_INTERNAL_SOURCE.`USD`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_exchange_rate_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_exchange_rate_hashid`,`new_column` = DBT_INTERNAL_SOURCE.`new_column` + + + when not matched then insert + (`id`, `currency`, `date`, `timestamp_col`, `HKD_special___characters`, `NZD`, `USD`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_exchange_rate_hashid`, `new_column`) + values + (`id`, `currency`, `date`, `timestamp_col`, `HKD_special___characters`, `NZD`, `USD`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_exchange_rate_hashid`, `new_column`) + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..5f8e9254f0ed --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,27 @@ + + + + + + + + merge into `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded` as DBT_INTERNAL_DEST + using ( + select * from `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded__dbt_tmp` + ) as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._airbyte_unique_key = DBT_INTERNAL_DEST._airbyte_unique_key + + + + when matched then update set + `_airbyte_unique_key` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key`,`id` = DBT_INTERNAL_SOURCE.`id`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_renamed_dedup_cdc_excluded_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_renamed_dedup_cdc_excluded_hashid`,`name` = DBT_INTERNAL_SOURCE.`name`,`_ab_cdc_lsn` = DBT_INTERNAL_SOURCE.`_ab_cdc_lsn`,`_ab_cdc_updated_at` = DBT_INTERNAL_SOURCE.`_ab_cdc_updated_at`,`_ab_cdc_deleted_at` = DBT_INTERNAL_SOURCE.`_ab_cdc_deleted_at` + + + when not matched then insert + (`_airbyte_unique_key`, `id`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_renamed_dedup_cdc_excluded_hashid`, `name`, `_ab_cdc_lsn`, `_ab_cdc_updated_at`, `_ab_cdc_deleted_at`) + values + (`_airbyte_unique_key`, `id`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_renamed_dedup_cdc_excluded_hashid`, `name`, `_ab_cdc_lsn`, `_ab_cdc_updated_at`, `_ab_cdc_deleted_at`) + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..ad91ec4869a2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,32 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(currency as + string +), ''), '-', coalesce(cast(new_column as + string +), ''), '-', coalesce(cast(date as + string +), ''), '-', coalesce(cast(timestamp_col as + string +), ''), '-', coalesce(cast(HKD_special___characters as + string +), ''), '-', coalesce(cast(NZD as + string +), ''), '-', coalesce(cast(USD as + string +), '')) as + string +))) as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab2` tmp +-- dedup_exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..3478ba52a70a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,26 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(name as + string +), ''), '-', coalesce(cast(_ab_cdc_lsn as + string +), ''), '-', coalesce(cast(_ab_cdc_updated_at as + string +), ''), '-', coalesce(cast(_ab_cdc_deleted_at as + string +), '')) as + string +))) as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2` tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql deleted file mode 100644 index 7c1ad19a5730..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ /dev/null @@ -1,12 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_array_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_query(_airbyte_data, ''$."conflict_stream_array"'') as conflict_stream_array, - _airbyte_emitted_at -from "test_normalization".test_normalization._airbyte_raw_conflict_stream_array as table_alias --- conflict_stream_array - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql deleted file mode 100644 index 643bff0b2550..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ /dev/null @@ -1,13 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_array_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - VARCHAR(max)) as id, - conflict_stream_array, - _airbyte_emitted_at -from "test_normalization"._airbyte_test_normalization."conflict_stream_array_ab1" --- conflict_stream_array - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql deleted file mode 100644 index 5974c570b808..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_array_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''), ''-'', coalesce(cast(cast(conflict_stream_array as - VARCHAR(max)) as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_array_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."conflict_stream_array_ab2" tmp --- conflict_stream_array - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql deleted file mode 100644 index f566cefad326..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql +++ /dev/null @@ -1,13 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_2_hashid, - json_value(conflict_stream_name, ''$."groups"'') as groups, - _airbyte_emitted_at -from "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name" as table_alias -where conflict_stream_name is not null --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql deleted file mode 100644 index e3fc0d6ddeb9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql +++ /dev/null @@ -1,13 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_2_hashid, - cast(groups as - VARCHAR(max)) as groups, - _airbyte_emitted_at -from "test_normalization"._airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab1" --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql deleted file mode 100644 index 659c21a38d23..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as - VARCHAR(max)), ''''), ''-'', coalesce(cast(groups as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_name_3_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab2" tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql deleted file mode 100644 index ea26e4e2cb68..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ /dev/null @@ -1,12 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_name_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_query(_airbyte_data, ''$."conflict_stream_name"'') as conflict_stream_name, - _airbyte_emitted_at -from "test_normalization".test_normalization._airbyte_raw_conflict_stream_name as table_alias --- conflict_stream_name - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql deleted file mode 100644 index c3cea18602f5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ /dev/null @@ -1,13 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_name_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - VARCHAR(max)) as id, - cast(conflict_stream_name as VARCHAR(max)) as conflict_stream_name, - _airbyte_emitted_at -from "test_normalization"._airbyte_test_normalization."conflict_stream_name_ab1" --- conflict_stream_name - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql deleted file mode 100644 index ba1cd49fb57a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_name_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''), ''-'', coalesce(cast(conflict_stream_name as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_name_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."conflict_stream_name_ab2" tmp --- conflict_stream_name - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql deleted file mode 100644 index e1c3ad365c80..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql +++ /dev/null @@ -1,13 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_hashid, - json_query(conflict_stream_name, ''$."conflict_stream_name"'') as conflict_stream_name, - _airbyte_emitted_at -from "test_normalization".test_normalization."conflict_stream_name" as table_alias -where conflict_stream_name is not null --- conflict_stream_name at conflict_stream_name/conflict_stream_name - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql deleted file mode 100644 index 412eb35553b9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql +++ /dev/null @@ -1,12 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_hashid, - cast(conflict_stream_name as VARCHAR(max)) as conflict_stream_name, - _airbyte_emitted_at -from "test_normalization"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab1" --- conflict_stream_name at conflict_stream_name/conflict_stream_name - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql deleted file mode 100644 index f2e51c42800e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(_airbyte_conflict_stream_name_hashid as - VARCHAR(max)), ''''), ''-'', coalesce(cast(conflict_stream_name as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_name_2_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab2" tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql deleted file mode 100644 index 0a2f7b877432..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ /dev/null @@ -1,12 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_scalar_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_value(_airbyte_data, ''$."conflict_stream_scalar"'') as conflict_stream_scalar, - _airbyte_emitted_at -from "test_normalization".test_normalization._airbyte_raw_conflict_stream_scalar as table_alias --- conflict_stream_scalar - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql deleted file mode 100644 index 0f158694bcd7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ /dev/null @@ -1,15 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_scalar_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - VARCHAR(max)) as id, - cast(conflict_stream_scalar as - bigint -) as conflict_stream_scalar, - _airbyte_emitted_at -from "test_normalization"._airbyte_test_normalization."conflict_stream_scalar_ab1" --- conflict_stream_scalar - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql deleted file mode 100644 index a11b99ab288b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_scalar_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''), ''-'', coalesce(cast(conflict_stream_scalar as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_scalar_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."conflict_stream_scalar_ab2" tmp --- conflict_stream_scalar - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql deleted file mode 100644 index 280ca6054748..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql +++ /dev/null @@ -1,15 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_nested_strea__nto_long_names_hashid, - json_query("partition", ''$."double_array_data"'') as double_array_data, - json_query("partition", ''$."DATA"'') as "DATA", - json_query("partition", ''$."column`_''''with\"_quotes"'') as "column`_''with""_quotes", - _airbyte_emitted_at -from "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names" as table_alias -where "partition" is not null --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql deleted file mode 100644 index f36721488e1c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql +++ /dev/null @@ -1,14 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_nested_strea__nto_long_names_hashid, - double_array_data, - "DATA", - "column`_''with""_quotes", - _airbyte_emitted_at -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab1" --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql deleted file mode 100644 index a65b3a07f7d1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql +++ /dev/null @@ -1,23 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(_airbyte_nested_strea__nto_long_names_hashid as - VARCHAR(max)), ''''), ''-'', coalesce(cast(cast(double_array_data as - VARCHAR(max)) as - VARCHAR(max)), ''''), ''-'', coalesce(cast(cast("DATA" as - VARCHAR(max)) as - VARCHAR(max)), ''''), ''-'', coalesce(cast(cast("column`_''with""_quotes" as - VARCHAR(max)) as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_partition_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab2" tmp --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql deleted file mode 100644 index b70ed12d5967..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql +++ /dev/null @@ -1,23 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_partition_hashid, - json_value( - "DATA".value, ''$."currency"'') as currency, - _airbyte_emitted_at -from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" as table_alias - - CROSS APPLY ( - SELECT [value] = CASE - WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) - WHEN [type] = 5 THEN [value] - END - FROM OPENJSON("DATA") - ) AS "DATA" -where "DATA" is not null --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql deleted file mode 100644 index d6d994472897..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql +++ /dev/null @@ -1,13 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as - VARCHAR(max)) as currency, - _airbyte_emitted_at -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab1" --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql deleted file mode 100644 index 0c24bc75d1af..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(_airbyte_partition_hashid as - VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_data_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab2" tmp --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql deleted file mode 100644 index 5aed664a26d4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql +++ /dev/null @@ -1,23 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_partition_hashid, - json_value( - "column`_''with""_quotes".value, ''$."currency"'') as currency, - _airbyte_emitted_at -from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" as table_alias - - CROSS APPLY ( - SELECT [value] = CASE - WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) - WHEN [type] = 5 THEN [value] - END - FROM OPENJSON("column`_''with""_quotes") - ) AS "column`_''with""_quotes" -where "column`_''with""_quotes" is not null --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql deleted file mode 100644 index de1463bf2bfa..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql +++ /dev/null @@ -1,13 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as - VARCHAR(max)) as currency, - _airbyte_emitted_at -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab1" --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql deleted file mode 100644 index 4980782957c7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(_airbyte_partition_hashid as - VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_column___with__quotes_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab2" tmp --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql deleted file mode 100644 index b427df0d0c57..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql +++ /dev/null @@ -1,23 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_partition_hashid, - json_value( - double_array_data.value, ''$."id"'') as id, - _airbyte_emitted_at -from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" as table_alias - - CROSS APPLY ( - SELECT [value] = CASE - WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) - WHEN [type] = 5 THEN [value] - END - FROM OPENJSON(double_array_data) - ) AS double_array_data -where double_array_data is not null --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql deleted file mode 100644 index 7b1302aa5bf2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql +++ /dev/null @@ -1,13 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(id as - VARCHAR(max)) as id, - _airbyte_emitted_at -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab1" --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql deleted file mode 100644 index 410371dfe86a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(_airbyte_partition_hashid as - VARCHAR(max)), ''''), ''-'', coalesce(cast(id as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_double_array_data_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab2" tmp --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql deleted file mode 100644 index fdffccf99ecd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql +++ /dev/null @@ -1,13 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_value(_airbyte_data, ''$."date"'') as "date", - json_query(_airbyte_data, ''$."partition"'') as "partition", - _airbyte_emitted_at -from "test_normalization".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias --- nested_stream_with_co__lting_into_long_names - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql deleted file mode 100644 index 3143ebbdee71..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql +++ /dev/null @@ -1,15 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - VARCHAR(max)) as id, - cast("date" as - VARCHAR(max)) as "date", - cast("partition" as VARCHAR(max)) as "partition", - _airbyte_emitted_at -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab1" --- nested_stream_with_co__lting_into_long_names - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql deleted file mode 100644 index 757103dd8f36..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as - VARCHAR(max)), ''''), ''-'', coalesce(cast("partition" as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_nested_strea__nto_long_names_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab2" tmp --- nested_stream_with_co__lting_into_long_names - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab4.sql deleted file mode 100644 index dd027fe5ce1f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab4.sql +++ /dev/null @@ -1,14 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab4__dbt_tmp" as - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_nested_strea__nto_long_names_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab3" tmp --- nested_stream_with_co__lting_into_long_names from "test_normalization".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql deleted file mode 100644 index 8c0f6ca2ce70..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql +++ /dev/null @@ -1,12 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_value(_airbyte_data, ''$."date"'') as "date", - _airbyte_emitted_at -from "test_normalization".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias --- non_nested_stream_wit__lting_into_long_names - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql deleted file mode 100644 index e72c7eb4bf73..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql +++ /dev/null @@ -1,14 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - VARCHAR(max)) as id, - cast("date" as - VARCHAR(max)) as "date", - _airbyte_emitted_at -from "test_normalization"._airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab1" --- non_nested_stream_wit__lting_into_long_names - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql deleted file mode 100644 index f2bb2f973065..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_non_nested_s__nto_long_names_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab2" tmp --- non_nested_stream_wit__lting_into_long_names - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab1.sql deleted file mode 100644 index 19133cf1718b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ /dev/null @@ -1,12 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."unnest_alias_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_query(_airbyte_data, ''$."children"'') as children, - _airbyte_emitted_at -from "test_normalization".test_normalization._airbyte_raw_unnest_alias as table_alias --- unnest_alias - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab2.sql deleted file mode 100644 index f6ffc45cd9bf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ /dev/null @@ -1,14 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."unnest_alias_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - children, - _airbyte_emitted_at -from "test_normalization"._airbyte_test_normalization."unnest_alias_ab1" --- unnest_alias - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab3.sql deleted file mode 100644 index b8e254e1c0d0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."unnest_alias_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''), ''-'', coalesce(cast(cast(children as - VARCHAR(max)) as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_unnest_alias_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."unnest_alias_ab2" tmp --- unnest_alias - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql deleted file mode 100644 index 0664fd667ab0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ /dev/null @@ -1,25 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."unnest_alias_children_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_unnest_alias_hashid, - json_value( - children.value, ''$."ab_id"'') as ab_id, - json_query( - children.value, ''$."owner"'') as owner, - _airbyte_emitted_at -from "test_normalization".test_normalization."unnest_alias" as table_alias - - CROSS APPLY ( - SELECT [value] = CASE - WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) - WHEN [type] = 5 THEN [value] - END - FROM OPENJSON(children) - ) AS children -where children is not null --- children at unnest_alias/children - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql deleted file mode 100644 index ccf13bfd4f71..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ /dev/null @@ -1,15 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."unnest_alias_children_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_unnest_alias_hashid, - cast(ab_id as - bigint -) as ab_id, - cast(owner as VARCHAR(max)) as owner, - _airbyte_emitted_at -from "test_normalization"._airbyte_test_normalization."unnest_alias_children_ab1" --- children at unnest_alias/children - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql deleted file mode 100644 index 8114b1045216..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."unnest_alias_children_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(_airbyte_unnest_alias_hashid as - VARCHAR(max)), ''''), ''-'', coalesce(cast(ab_id as - VARCHAR(max)), ''''), ''-'', coalesce(cast(owner as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_children_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."unnest_alias_children_ab2" tmp --- children at unnest_alias/children - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql deleted file mode 100644 index 6a01d606373f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ /dev/null @@ -1,13 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."unnest_alias_children_owner_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_children_hashid, - json_value(owner, ''$."owner_id"'') as owner_id, - _airbyte_emitted_at -from "test_normalization".test_normalization."unnest_alias_children" as table_alias -where owner is not null --- owner at unnest_alias/children/owner - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql deleted file mode 100644 index 9f77ba923a2b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ /dev/null @@ -1,14 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."unnest_alias_children_owner_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_children_hashid, - cast(owner_id as - bigint -) as owner_id, - _airbyte_emitted_at -from "test_normalization"._airbyte_test_normalization."unnest_alias_children_owner_ab1" --- owner at unnest_alias/children/owner - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql deleted file mode 100644 index fb293fff58eb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."unnest_alias_children_owner_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(_airbyte_children_hashid as - VARCHAR(max)), ''''), ''-'', coalesce(cast(owner_id as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_owner_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."unnest_alias_children_owner_ab2" tmp --- owner at unnest_alias/children/owner - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql deleted file mode 100644 index acb59a7d8af0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql +++ /dev/null @@ -1,12 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_value(_airbyte_data, ''$."date"'') as "date", - _airbyte_emitted_at -from "test_normalization".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias --- simple_stream_with_na__lting_into_long_names - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql deleted file mode 100644 index df95c421d630..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql +++ /dev/null @@ -1,14 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - VARCHAR(max)) as id, - cast("date" as - VARCHAR(max)) as "date", - _airbyte_emitted_at -from "test_normalization"._airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab1" --- simple_stream_with_na__lting_into_long_names - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql deleted file mode 100644 index 3578b978cc3d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_simple_strea__nto_long_names_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab2" tmp --- simple_stream_with_na__lting_into_long_names - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql deleted file mode 100644 index 105849dc8988..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql +++ /dev/null @@ -1,55 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_array__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_array__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_array__dbt_tmp"','U') is not null - begin - drop table test_normalization."conflict_stream_array__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."conflict_stream_array__dbt_tmp_temp_view" as - --- Final base SQL model -select - id, - conflict_stream_array, - _airbyte_emitted_at, - _airbyte_conflict_stream_array_hashid -from "test_normalization"._airbyte_test_normalization."conflict_stream_array_ab3" --- conflict_stream_array from "test_normalization".test_normalization._airbyte_raw_conflict_stream_array - '); - - SELECT * INTO "test_normalization".test_normalization."conflict_stream_array__dbt_tmp" FROM - "test_normalization".test_normalization."conflict_stream_array__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_array__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_array__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_conflict_stream_array__dbt_tmp_cci' - AND object_id=object_id('test_normalization_conflict_stream_array__dbt_tmp') - ) - DROP index test_normalization.conflict_stream_array__dbt_tmp.test_normalization_conflict_stream_array__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_array__dbt_tmp_cci - ON test_normalization.conflict_stream_array__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql deleted file mode 100644 index 859508823935..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql +++ /dev/null @@ -1,55 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_name__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_name__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_name__dbt_tmp"','U') is not null - begin - drop table test_normalization."conflict_stream_name__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."conflict_stream_name__dbt_tmp_temp_view" as - --- Final base SQL model -select - id, - conflict_stream_name, - _airbyte_emitted_at, - _airbyte_conflict_stream_name_hashid -from "test_normalization"._airbyte_test_normalization."conflict_stream_name_ab3" --- conflict_stream_name from "test_normalization".test_normalization._airbyte_raw_conflict_stream_name - '); - - SELECT * INTO "test_normalization".test_normalization."conflict_stream_name__dbt_tmp" FROM - "test_normalization".test_normalization."conflict_stream_name__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_name__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_name__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_conflict_stream_name__dbt_tmp_cci' - AND object_id=object_id('test_normalization_conflict_stream_name__dbt_tmp') - ) - DROP index test_normalization.conflict_stream_name__dbt_tmp.test_normalization_conflict_stream_name__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_name__dbt_tmp_cci - ON test_normalization.conflict_stream_name__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql deleted file mode 100644 index 45c332535f17..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql +++ /dev/null @@ -1,55 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp"','U') is not null - begin - drop table test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view" as - --- Final base SQL model -select - _airbyte_conflict_stream_name_2_hashid, - groups, - _airbyte_emitted_at, - _airbyte_conflict_stream_name_3_hashid -from "test_normalization"._airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab3" --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name" - '); - - SELECT * INTO "test_normalization".test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp" FROM - "test_normalization".test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_conflict_stream_name____conflict_stream_name__dbt_tmp_cci' - AND object_id=object_id('test_normalization_conflict_stream_name____conflict_stream_name__dbt_tmp') - ) - DROP index test_normalization.conflict_stream_name____conflict_stream_name__dbt_tmp.test_normalization_conflict_stream_name____conflict_stream_name__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_name____conflict_stream_name__dbt_tmp_cci - ON test_normalization.conflict_stream_name____conflict_stream_name__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql deleted file mode 100644 index a1bc83fb3a97..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ /dev/null @@ -1,55 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp"','U') is not null - begin - drop table test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view" as - --- Final base SQL model -select - _airbyte_conflict_stream_name_hashid, - conflict_stream_name, - _airbyte_emitted_at, - _airbyte_conflict_stream_name_2_hashid -from "test_normalization"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab3" --- conflict_stream_name at conflict_stream_name/conflict_stream_name from "test_normalization".test_normalization."conflict_stream_name" - '); - - SELECT * INTO "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp" FROM - "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_conflict_stream_name_conflict_stream_name__dbt_tmp_cci' - AND object_id=object_id('test_normalization_conflict_stream_name_conflict_stream_name__dbt_tmp') - ) - DROP index test_normalization.conflict_stream_name_conflict_stream_name__dbt_tmp.test_normalization_conflict_stream_name_conflict_stream_name__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_name_conflict_stream_name__dbt_tmp_cci - ON test_normalization.conflict_stream_name_conflict_stream_name__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql deleted file mode 100644 index ca7145c9cea6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ /dev/null @@ -1,55 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_scalar__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_scalar__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_scalar__dbt_tmp"','U') is not null - begin - drop table test_normalization."conflict_stream_scalar__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."conflict_stream_scalar__dbt_tmp_temp_view" as - --- Final base SQL model -select - id, - conflict_stream_scalar, - _airbyte_emitted_at, - _airbyte_conflict_stream_scalar_hashid -from "test_normalization"._airbyte_test_normalization."conflict_stream_scalar_ab3" --- conflict_stream_scalar from "test_normalization".test_normalization._airbyte_raw_conflict_stream_scalar - '); - - SELECT * INTO "test_normalization".test_normalization."conflict_stream_scalar__dbt_tmp" FROM - "test_normalization".test_normalization."conflict_stream_scalar__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_scalar__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_scalar__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_conflict_stream_scalar__dbt_tmp_cci' - AND object_id=object_id('test_normalization_conflict_stream_scalar__dbt_tmp') - ) - DROP index test_normalization.conflict_stream_scalar__dbt_tmp.test_normalization_conflict_stream_scalar__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_scalar__dbt_tmp_cci - ON test_normalization.conflict_stream_scalar__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql deleted file mode 100644 index f8ba28359065..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql +++ /dev/null @@ -1,57 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp"','U') is not null - begin - drop table test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view" as - --- Final base SQL model -select - _airbyte_nested_strea__nto_long_names_hashid, - double_array_data, - "DATA", - "column`_''with""_quotes", - _airbyte_emitted_at, - _airbyte_partition_hashid -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab3" --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names" - '); - - SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp" FROM - "test_normalization".test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_nested_stream_with_co___long_names_partition__dbt_tmp_cci' - AND object_id=object_id('test_normalization_nested_stream_with_co___long_names_partition__dbt_tmp') - ) - DROP index test_normalization.nested_stream_with_co___long_names_partition__dbt_tmp.test_normalization_nested_stream_with_co___long_names_partition__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co___long_names_partition__dbt_tmp_cci - ON test_normalization.nested_stream_with_co___long_names_partition__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql deleted file mode 100644 index a2ffbe6664b6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql +++ /dev/null @@ -1,55 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp"','U') is not null - begin - drop table test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view" as - --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_emitted_at, - _airbyte_data_hashid -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab3" --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" - '); - - SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp" FROM - "test_normalization".test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_nested_stream_with_co___names_partition_data__dbt_tmp_cci' - AND object_id=object_id('test_normalization_nested_stream_with_co___names_partition_data__dbt_tmp') - ) - DROP index test_normalization.nested_stream_with_co___names_partition_data__dbt_tmp.test_normalization_nested_stream_with_co___names_partition_data__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co___names_partition_data__dbt_tmp_cci - ON test_normalization.nested_stream_with_co___names_partition_data__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql deleted file mode 100644 index 7321ee6468d9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql +++ /dev/null @@ -1,55 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp"','U') is not null - begin - drop table test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view" as - --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_emitted_at, - _airbyte_column___with__quotes_hashid -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab3" --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" - '); - - SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp" FROM - "test_normalization".test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_nested_stream_with_co__column___with__quotes__dbt_tmp_cci' - AND object_id=object_id('test_normalization_nested_stream_with_co__column___with__quotes__dbt_tmp') - ) - DROP index test_normalization.nested_stream_with_co__column___with__quotes__dbt_tmp.test_normalization_nested_stream_with_co__column___with__quotes__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co__column___with__quotes__dbt_tmp_cci - ON test_normalization.nested_stream_with_co__column___with__quotes__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql deleted file mode 100644 index a6a8e1aaa8ce..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql +++ /dev/null @@ -1,55 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp"','U') is not null - begin - drop table test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view" as - --- Final base SQL model -select - _airbyte_partition_hashid, - id, - _airbyte_emitted_at, - _airbyte_double_array_data_hashid -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab3" --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" - '); - - SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp" FROM - "test_normalization".test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_nested_stream_with_co__ion_double_array_data__dbt_tmp_cci' - AND object_id=object_id('test_normalization_nested_stream_with_co__ion_double_array_data__dbt_tmp') - ) - DROP index test_normalization.nested_stream_with_co__ion_double_array_data__dbt_tmp.test_normalization_nested_stream_with_co__ion_double_array_data__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co__ion_double_array_data__dbt_tmp_cci - ON test_normalization.nested_stream_with_co__ion_double_array_data__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql deleted file mode 100644 index dbb47b24e2ad..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql +++ /dev/null @@ -1,57 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co__lting_into_long_names__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names__dbt_tmp"','U') is not null - begin - drop table test_normalization."nested_stream_with_co__lting_into_long_names__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."nested_stream_with_co__lting_into_long_names__dbt_tmp_temp_view" as - --- Final base SQL model -select - id, - "date", - "partition", - _airbyte_emitted_at, - _airbyte_nested_strea__nto_long_names_hashid -from "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names_scd" --- nested_stream_with_co__lting_into_long_names from "test_normalization".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names -where _airbyte_active_row = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names__dbt_tmp" FROM - "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co__lting_into_long_names__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_nested_stream_with_co__lting_into_long_names__dbt_tmp_cci' - AND object_id=object_id('test_normalization_nested_stream_with_co__lting_into_long_names__dbt_tmp') - ) - DROP index test_normalization.nested_stream_with_co__lting_into_long_names__dbt_tmp.test_normalization_nested_stream_with_co__lting_into_long_names__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co__lting_into_long_names__dbt_tmp_cci - ON test_normalization.nested_stream_with_co__lting_into_long_names__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql deleted file mode 100644 index e3f6c521c475..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql +++ /dev/null @@ -1,66 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names_scd__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co__lting_into_long_names_scd__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names_scd__dbt_tmp"','U') is not null - begin - drop table test_normalization."nested_stream_with_co__lting_into_long_names_scd__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."nested_stream_with_co__lting_into_long_names_scd__dbt_tmp_temp_view" as - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - "date", - "partition", - "date" as _airbyte_start_at, - lag("date") over ( - partition by id - order by "date" desc, "date" desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag("date") over ( - partition by id - order by "date" desc, "date" desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_nested_strea__nto_long_names_hashid -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab4" --- nested_stream_with_co__lting_into_long_names from "test_normalization".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names -where _airbyte_row_num = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names_scd__dbt_tmp" FROM - "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names_scd__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names_scd__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co__lting_into_long_names_scd__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_nested_stream_with_co__lting_into_long_names_scd__dbt_tmp_cci' - AND object_id=object_id('test_normalization_nested_stream_with_co__lting_into_long_names_scd__dbt_tmp') - ) - DROP index test_normalization.nested_stream_with_co__lting_into_long_names_scd__dbt_tmp.test_normalization_nested_stream_with_co__lting_into_long_names_scd__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co__lting_into_long_names_scd__dbt_tmp_cci - ON test_normalization.nested_stream_with_co__lting_into_long_names_scd__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql deleted file mode 100644 index 9508a59004b2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql +++ /dev/null @@ -1,55 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp"','U') is not null - begin - drop table test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view" as - --- Final base SQL model -select - id, - "date", - _airbyte_emitted_at, - _airbyte_non_nested_s__nto_long_names_hashid -from "test_normalization"._airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab3" --- non_nested_stream_wit__lting_into_long_names from "test_normalization".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names - '); - - SELECT * INTO "test_normalization".test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp" FROM - "test_normalization".test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_non_nested_stream_wit__lting_into_long_names__dbt_tmp_cci' - AND object_id=object_id('test_normalization_non_nested_stream_wit__lting_into_long_names__dbt_tmp') - ) - DROP index test_normalization.non_nested_stream_wit__lting_into_long_names__dbt_tmp.test_normalization_non_nested_stream_wit__lting_into_long_names__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_non_nested_stream_wit__lting_into_long_names__dbt_tmp_cci - ON test_normalization.non_nested_stream_wit__lting_into_long_names__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql deleted file mode 100644 index 314f2b44b400..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,55 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."unnest_alias__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."unnest_alias__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."unnest_alias__dbt_tmp"','U') is not null - begin - drop table test_normalization."unnest_alias__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."unnest_alias__dbt_tmp_temp_view" as - --- Final base SQL model -select - id, - children, - _airbyte_emitted_at, - _airbyte_unnest_alias_hashid -from "test_normalization"._airbyte_test_normalization."unnest_alias_ab3" --- unnest_alias from "test_normalization".test_normalization._airbyte_raw_unnest_alias - '); - - SELECT * INTO "test_normalization".test_normalization."unnest_alias__dbt_tmp" FROM - "test_normalization".test_normalization."unnest_alias__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."unnest_alias__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."unnest_alias__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_unnest_alias__dbt_tmp_cci' - AND object_id=object_id('test_normalization_unnest_alias__dbt_tmp') - ) - DROP index test_normalization.unnest_alias__dbt_tmp.test_normalization_unnest_alias__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_unnest_alias__dbt_tmp_cci - ON test_normalization.unnest_alias__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql deleted file mode 100644 index 071aae66aeee..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql +++ /dev/null @@ -1,56 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."unnest_alias_children__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."unnest_alias_children__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."unnest_alias_children__dbt_tmp"','U') is not null - begin - drop table test_normalization."unnest_alias_children__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."unnest_alias_children__dbt_tmp_temp_view" as - --- Final base SQL model -select - _airbyte_unnest_alias_hashid, - ab_id, - owner, - _airbyte_emitted_at, - _airbyte_children_hashid -from "test_normalization"._airbyte_test_normalization."unnest_alias_children_ab3" --- children at unnest_alias/children from "test_normalization".test_normalization."unnest_alias" - '); - - SELECT * INTO "test_normalization".test_normalization."unnest_alias_children__dbt_tmp" FROM - "test_normalization".test_normalization."unnest_alias_children__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."unnest_alias_children__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."unnest_alias_children__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_unnest_alias_children__dbt_tmp_cci' - AND object_id=object_id('test_normalization_unnest_alias_children__dbt_tmp') - ) - DROP index test_normalization.unnest_alias_children__dbt_tmp.test_normalization_unnest_alias_children__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_unnest_alias_children__dbt_tmp_cci - ON test_normalization.unnest_alias_children__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql deleted file mode 100644 index 8178adafcd0d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ /dev/null @@ -1,55 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."unnest_alias_children_owner__dbt_tmp"','U') is not null - begin - drop table test_normalization."unnest_alias_children_owner__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view" as - --- Final base SQL model -select - _airbyte_children_hashid, - owner_id, - _airbyte_emitted_at, - _airbyte_owner_hashid -from "test_normalization"._airbyte_test_normalization."unnest_alias_children_owner_ab3" --- owner at unnest_alias/children/owner from "test_normalization".test_normalization."unnest_alias_children" - '); - - SELECT * INTO "test_normalization".test_normalization."unnest_alias_children_owner__dbt_tmp" FROM - "test_normalization".test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_unnest_alias_children_owner__dbt_tmp_cci' - AND object_id=object_id('test_normalization_unnest_alias_children_owner__dbt_tmp') - ) - DROP index test_normalization.unnest_alias_children_owner__dbt_tmp.test_normalization_unnest_alias_children_owner__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_unnest_alias_children_owner__dbt_tmp_cci - ON test_normalization.unnest_alias_children_owner__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql deleted file mode 100644 index ab9761955c89..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql +++ /dev/null @@ -1,55 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp"','U') is not null - begin - drop table test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view" as - --- Final base SQL model -select - id, - "date", - _airbyte_emitted_at, - _airbyte_simple_strea__nto_long_names_hashid -from "test_normalization"._airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab3" --- simple_stream_with_na__lting_into_long_names from "test_normalization".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names - '); - - SELECT * INTO "test_normalization".test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp" FROM - "test_normalization".test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_namespace_simple_stream_with_na__lting_into_long_names__dbt_tmp_cci' - AND object_id=object_id('test_normalization_namespace_simple_stream_with_na__lting_into_long_names__dbt_tmp') - ) - DROP index test_normalization_namespace.simple_stream_with_na__lting_into_long_names__dbt_tmp.test_normalization_namespace_simple_stream_with_na__lting_into_long_names__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_namespace_simple_stream_with_na__lting_into_long_names__dbt_tmp_cci - ON test_normalization_namespace.simple_stream_with_na__lting_into_long_names__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql new file mode 100644 index 000000000000..1567f2b78797 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql @@ -0,0 +1,15 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_array_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_query(_airbyte_data, ''$."conflict_stream_array"'') as conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_conflict_stream_array as table_alias +-- conflict_stream_array +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql new file mode 100644 index 000000000000..a16d74e97ea4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_array_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + VARCHAR(max)) as id, + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."conflict_stream_array_ab1" +-- conflict_stream_array +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql new file mode 100644 index 000000000000..e5bde2f88eab --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql @@ -0,0 +1,20 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_array_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(cast(conflict_stream_array as + VARCHAR(max)) as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_array_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."conflict_stream_array_ab2" tmp +-- conflict_stream_array +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..711edc45508d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_2_hashid, + json_value(conflict_stream_name, ''$."groups"'') as groups, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name" as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..a8f6a33ad157 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_2_hashid, + cast(groups as + VARCHAR(max)) as groups, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab1" +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..4c510332a6ee --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(groups as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_name_3_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab2" tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..8f1fc7fe16db --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql @@ -0,0 +1,15 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_name_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_query(_airbyte_data, ''$."conflict_stream_name"'') as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_conflict_stream_name as table_alias +-- conflict_stream_name +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..1e106f88c407 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_name_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + VARCHAR(max)) as id, + cast(conflict_stream_name as VARCHAR(max)) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."conflict_stream_name_ab1" +-- conflict_stream_name +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..dd136bf9bad9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_name_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(conflict_stream_name as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_name_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."conflict_stream_name_ab2" tmp +-- conflict_stream_name +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..26744bb811ae --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_hashid, + json_query(conflict_stream_name, ''$."conflict_stream_name"'') as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization."conflict_stream_name" as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..6a630d1e24c1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql @@ -0,0 +1,15 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_hashid, + cast(conflict_stream_name as VARCHAR(max)) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab1" +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..cbe3f02a60f7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_conflict_stream_name_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(conflict_stream_name as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_name_2_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab2" tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql new file mode 100644 index 000000000000..ecce4bc0d160 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql @@ -0,0 +1,15 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_scalar_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."conflict_stream_scalar"'') as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_conflict_stream_scalar as table_alias +-- conflict_stream_scalar +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql new file mode 100644 index 000000000000..0cb7a859994b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql @@ -0,0 +1,18 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_scalar_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + VARCHAR(max)) as id, + cast(conflict_stream_scalar as + bigint +) as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."conflict_stream_scalar_ab1" +-- conflict_stream_scalar +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql new file mode 100644 index 000000000000..54f208622b20 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_scalar_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(conflict_stream_scalar as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_scalar_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."conflict_stream_scalar_ab2" tmp +-- conflict_stream_scalar +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql new file mode 100644 index 000000000000..d74cd1c7389f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql @@ -0,0 +1,18 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_nested_strea__nto_long_names_hashid, + json_query("partition", ''$."double_array_data"'') as double_array_data, + json_query("partition", ''$."DATA"'') as "DATA", + json_query("partition", ''$."column`_''''with\"_quotes"'') as "column`_''with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names" as table_alias +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and "partition" is not null + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql new file mode 100644 index 000000000000..90c0eafb817d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql @@ -0,0 +1,17 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_nested_strea__nto_long_names_hashid, + double_array_data, + "DATA", + "column`_''with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab1" +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql new file mode 100644 index 000000000000..f898a92e06da --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql @@ -0,0 +1,24 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_nested_strea__nto_long_names_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(cast(double_array_data as + VARCHAR(max)) as + VARCHAR(max)), ''''), ''-'', coalesce(cast(cast("DATA" as + VARCHAR(max)) as + VARCHAR(max)), ''''), ''-'', coalesce(cast(cast("column`_''with""_quotes" as + VARCHAR(max)) as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_partition_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab2" tmp +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql new file mode 100644 index 000000000000..deb088fe4931 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql @@ -0,0 +1,26 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + json_value( + "DATA".value, ''$."currency"'') as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" as table_alias +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA + + CROSS APPLY ( + SELECT [value] = CASE + WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) + WHEN [type] = 5 THEN [value] + END + FROM OPENJSON("DATA") + ) AS "DATA" +where 1 = 1 +and "DATA" is not null + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql new file mode 100644 index 000000000000..505eb01f4d31 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as + VARCHAR(max)) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab1" +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql new file mode 100644 index 000000000000..89631a486695 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_partition_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_data_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab2" tmp +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql new file mode 100644 index 000000000000..96a6f2a9eacc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql @@ -0,0 +1,26 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + json_value( + "column`_''with""_quotes".value, ''$."currency"'') as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" as table_alias +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes + + CROSS APPLY ( + SELECT [value] = CASE + WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) + WHEN [type] = 5 THEN [value] + END + FROM OPENJSON("column`_''with""_quotes") + ) AS "column`_''with""_quotes" +where 1 = 1 +and "column`_''with""_quotes" is not null + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql new file mode 100644 index 000000000000..9edfb5ade031 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as + VARCHAR(max)) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab1" +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql new file mode 100644 index 000000000000..1da3ab107876 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_partition_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_column___with__quotes_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab2" tmp +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql new file mode 100644 index 000000000000..7e83428589ca --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql @@ -0,0 +1,26 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + json_value( + double_array_data.value, ''$."id"'') as id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" as table_alias +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data + + CROSS APPLY ( + SELECT [value] = CASE + WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) + WHEN [type] = 5 THEN [value] + END + FROM OPENJSON(double_array_data) + ) AS double_array_data +where 1 = 1 +and double_array_data is not null + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql new file mode 100644 index 000000000000..1eb3236e85d9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(id as + VARCHAR(max)) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab1" +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql new file mode 100644 index 000000000000..b6e73981f3fb --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_partition_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(id as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_double_array_data_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab2" tmp +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql new file mode 100644 index 000000000000..e921a062d3b7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql @@ -0,0 +1,17 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."date"'') as "date", + json_query(_airbyte_data, ''$."partition"'') as "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias +-- nested_stream_with_co__lting_into_long_names +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql new file mode 100644 index 000000000000..5eafd0d71a6b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + VARCHAR(max)) as id, + cast("date" as + VARCHAR(max)) as "date", + cast("partition" as VARCHAR(max)) as "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab1" +-- nested_stream_with_co__lting_into_long_names +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql new file mode 100644 index 000000000000..c81d35772445 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql @@ -0,0 +1,15 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."date"'') as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias +-- non_nested_stream_wit__lting_into_long_names +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql new file mode 100644 index 000000000000..4abb31531a75 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql @@ -0,0 +1,17 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + VARCHAR(max)) as id, + cast("date" as + VARCHAR(max)) as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab1" +-- non_nested_stream_wit__lting_into_long_names +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql new file mode 100644 index 000000000000..094bd24079d4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_non_nested_s__nto_long_names_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab2" tmp +-- non_nested_stream_wit__lting_into_long_names +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql new file mode 100644 index 000000000000..177d8bda7d07 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."unnest_alias_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_query(_airbyte_data, ''$."children"'') as children, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_unnest_alias as table_alias +-- unnest_alias +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql new file mode 100644 index 000000000000..a9ef564eff36 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql @@ -0,0 +1,18 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."unnest_alias_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + children, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."unnest_alias_ab1" +-- unnest_alias +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql new file mode 100644 index 000000000000..4544841f480f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql @@ -0,0 +1,21 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."unnest_alias_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(cast(children as + VARCHAR(max)) as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_unnest_alias_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."unnest_alias_ab2" tmp +-- unnest_alias +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql new file mode 100644 index 000000000000..d9f07f78b06c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql @@ -0,0 +1,28 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."unnest_alias_children_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_unnest_alias_hashid, + json_value( + children.value, ''$."ab_id"'') as ab_id, + json_query( + children.value, ''$."owner"'') as owner, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization."unnest_alias" as table_alias +-- children at unnest_alias/children + + CROSS APPLY ( + SELECT [value] = CASE + WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) + WHEN [type] = 5 THEN [value] + END + FROM OPENJSON(children) + ) AS children +where 1 = 1 +and children is not null + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql new file mode 100644 index 000000000000..c8d4bf47083c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql @@ -0,0 +1,18 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."unnest_alias_children_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_unnest_alias_hashid, + cast(ab_id as + bigint +) as ab_id, + cast(owner as VARCHAR(max)) as owner, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."unnest_alias_children_ab1" +-- children at unnest_alias/children +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql new file mode 100644 index 000000000000..6ad3a1c4caf8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql @@ -0,0 +1,20 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."unnest_alias_children_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_unnest_alias_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(ab_id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(owner as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_children_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."unnest_alias_children_ab2" tmp +-- children at unnest_alias/children +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql new file mode 100644 index 000000000000..7f49a4c7f4ea --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."unnest_alias_children_owner_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_children_hashid, + json_value(owner, ''$."owner_id"'') as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization."unnest_alias_children" as table_alias +-- owner at unnest_alias/children/owner +where 1 = 1 +and owner is not null + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql new file mode 100644 index 000000000000..9cf0dca5efb6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql @@ -0,0 +1,17 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."unnest_alias_children_owner_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_children_hashid, + cast(owner_id as + bigint +) as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."unnest_alias_children_owner_ab1" +-- owner at unnest_alias/children/owner +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql new file mode 100644 index 000000000000..fe2cfe2e2a98 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."unnest_alias_children_owner_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_children_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(owner_id as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_owner_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."unnest_alias_children_owner_ab2" tmp +-- owner at unnest_alias/children/owner +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql new file mode 100644 index 000000000000..a86efc7f2153 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql @@ -0,0 +1,15 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."date"'') as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias +-- simple_stream_with_na__lting_into_long_names +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql new file mode 100644 index 000000000000..077cbe05b163 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql @@ -0,0 +1,17 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + VARCHAR(max)) as id, + cast("date" as + VARCHAR(max)) as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab1" +-- simple_stream_with_na__lting_into_long_names +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql new file mode 100644 index 000000000000..f2f9167067b8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_simple_strea__nto_long_names_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab2" tmp +-- simple_stream_with_na__lting_into_long_names +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql new file mode 100644 index 000000000000..fae879eca8b1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql @@ -0,0 +1,125 @@ + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names_scd_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co__lting_into_long_names_scd_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names_scd"','U') is not null + begin + drop table test_normalization."nested_stream_with_co__lting_into_long_names_scd" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."nested_stream_with_co__lting_into_long_names_scd_temp_view" as + +with + +input_data as ( + select * + from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab3" + -- nested_stream_with_co__lting_into_long_names from "test_normalization".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_unique_key, + id, + "date", + "partition", + "date" as _airbyte_start_at, + lag("date") over ( + partition by id + order by + "date" desc, + "date" desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag("date") over ( + partition by id + order by + "date" desc, + "date" desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_nested_strea__nto_long_names_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_unique_key as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_start_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_emitted_at as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + "date", + "partition", + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_nested_strea__nto_long_names_hashid +from dedup_data where _airbyte_row_num = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names_scd" FROM + "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names_scd_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names_scd_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co__lting_into_long_names_scd_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_nested_stream_with_co__lting_into_long_names_scd_cci' + AND object_id=object_id('test_normalization_nested_stream_with_co__lting_into_long_names_scd') + ) + DROP index test_normalization.nested_stream_with_co__lting_into_long_names_scd.test_normalization_nested_stream_with_co__lting_into_long_names_scd_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co__lting_into_long_names_scd_cci + ON test_normalization.nested_stream_with_co__lting_into_long_names_scd + + + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql new file mode 100644 index 000000000000..eaa24483e4d7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql @@ -0,0 +1,65 @@ + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co__lting_into_long_names_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names"','U') is not null + begin + drop table test_normalization."nested_stream_with_co__lting_into_long_names" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."nested_stream_with_co__lting_into_long_names_temp_view" as + +-- Final base SQL model +select + _airbyte_unique_key, + id, + "date", + "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_nested_strea__nto_long_names_hashid +from "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names_scd" +-- nested_stream_with_co__lting_into_long_names from "test_normalization".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 +and _airbyte_active_row = 1 + + '); + + SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names" FROM + "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co__lting_into_long_names_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_nested_stream_with_co__lting_into_long_names_cci' + AND object_id=object_id('test_normalization_nested_stream_with_co__lting_into_long_names') + ) + DROP index test_normalization.nested_stream_with_co__lting_into_long_names.test_normalization_nested_stream_with_co__lting_into_long_names_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co__lting_into_long_names_cci + ON test_normalization.nested_stream_with_co__lting_into_long_names + + + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..8c21dfcb78df --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql @@ -0,0 +1,62 @@ + + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias_temp_view"','V') is not null + begin + drop view test_normalization."unnest_alias_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias"','U') is not null + begin + drop table test_normalization."unnest_alias" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."unnest_alias_temp_view" as + +-- Final base SQL model +select + id, + children, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_unnest_alias_hashid +from "test_normalization"._airbyte_test_normalization."unnest_alias_ab3" +-- unnest_alias from "test_normalization".test_normalization._airbyte_raw_unnest_alias +where 1 = 1 + + '); + + SELECT * INTO "test_normalization".test_normalization."unnest_alias" FROM + "test_normalization".test_normalization."unnest_alias_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias_temp_view"','V') is not null + begin + drop view test_normalization."unnest_alias_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_unnest_alias_cci' + AND object_id=object_id('test_normalization_unnest_alias') + ) + DROP index test_normalization.unnest_alias.test_normalization_unnest_alias_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_unnest_alias_cci + ON test_normalization.unnest_alias + + + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql new file mode 100644 index 000000000000..2ffefaacaa27 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_array__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_array__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_array__dbt_tmp"','U') is not null + begin + drop table test_normalization."conflict_stream_array__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."conflict_stream_array__dbt_tmp_temp_view" as + +-- Final base SQL model +select + id, + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_conflict_stream_array_hashid +from "test_normalization"._airbyte_test_normalization."conflict_stream_array_ab3" +-- conflict_stream_array from "test_normalization".test_normalization._airbyte_raw_conflict_stream_array +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."conflict_stream_array__dbt_tmp" FROM + "test_normalization".test_normalization."conflict_stream_array__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_array__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_array__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_conflict_stream_array__dbt_tmp_cci' + AND object_id=object_id('test_normalization_conflict_stream_array__dbt_tmp') + ) + DROP index test_normalization.conflict_stream_array__dbt_tmp.test_normalization_conflict_stream_array__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_array__dbt_tmp_cci + ON test_normalization.conflict_stream_array__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql new file mode 100644 index 000000000000..e2c0ff6666d5 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_name__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name__dbt_tmp"','U') is not null + begin + drop table test_normalization."conflict_stream_name__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."conflict_stream_name__dbt_tmp_temp_view" as + +-- Final base SQL model +select + id, + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_hashid +from "test_normalization"._airbyte_test_normalization."conflict_stream_name_ab3" +-- conflict_stream_name from "test_normalization".test_normalization._airbyte_raw_conflict_stream_name +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."conflict_stream_name__dbt_tmp" FROM + "test_normalization".test_normalization."conflict_stream_name__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_name__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_conflict_stream_name__dbt_tmp_cci' + AND object_id=object_id('test_normalization_conflict_stream_name__dbt_tmp') + ) + DROP index test_normalization.conflict_stream_name__dbt_tmp.test_normalization_conflict_stream_name__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_name__dbt_tmp_cci + ON test_normalization.conflict_stream_name__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql new file mode 100644 index 000000000000..cc5f9ded53f2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp"','U') is not null + begin + drop table test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view" as + +-- Final base SQL model +select + _airbyte_conflict_stream_name_2_hashid, + groups, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_3_hashid +from "test_normalization"._airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab3" +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name" +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp" FROM + "test_normalization".test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_conflict_stream_name____conflict_stream_name__dbt_tmp_cci' + AND object_id=object_id('test_normalization_conflict_stream_name____conflict_stream_name__dbt_tmp') + ) + DROP index test_normalization.conflict_stream_name____conflict_stream_name__dbt_tmp.test_normalization_conflict_stream_name____conflict_stream_name__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_name____conflict_stream_name__dbt_tmp_cci + ON test_normalization.conflict_stream_name____conflict_stream_name__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql new file mode 100644 index 000000000000..79de98cb1242 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp"','U') is not null + begin + drop table test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view" as + +-- Final base SQL model +select + _airbyte_conflict_stream_name_hashid, + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_2_hashid +from "test_normalization"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab3" +-- conflict_stream_name at conflict_stream_name/conflict_stream_name from "test_normalization".test_normalization."conflict_stream_name" +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp" FROM + "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_conflict_stream_name_conflict_stream_name__dbt_tmp_cci' + AND object_id=object_id('test_normalization_conflict_stream_name_conflict_stream_name__dbt_tmp') + ) + DROP index test_normalization.conflict_stream_name_conflict_stream_name__dbt_tmp.test_normalization_conflict_stream_name_conflict_stream_name__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_name_conflict_stream_name__dbt_tmp_cci + ON test_normalization.conflict_stream_name_conflict_stream_name__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql new file mode 100644 index 000000000000..62ee6566a04a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_scalar__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_scalar__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_scalar__dbt_tmp"','U') is not null + begin + drop table test_normalization."conflict_stream_scalar__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."conflict_stream_scalar__dbt_tmp_temp_view" as + +-- Final base SQL model +select + id, + conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_conflict_stream_scalar_hashid +from "test_normalization"._airbyte_test_normalization."conflict_stream_scalar_ab3" +-- conflict_stream_scalar from "test_normalization".test_normalization._airbyte_raw_conflict_stream_scalar +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."conflict_stream_scalar__dbt_tmp" FROM + "test_normalization".test_normalization."conflict_stream_scalar__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_scalar__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_scalar__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_conflict_stream_scalar__dbt_tmp_cci' + AND object_id=object_id('test_normalization_conflict_stream_scalar__dbt_tmp') + ) + DROP index test_normalization.conflict_stream_scalar__dbt_tmp.test_normalization_conflict_stream_scalar__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_scalar__dbt_tmp_cci + ON test_normalization.conflict_stream_scalar__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql new file mode 100644 index 000000000000..a1277fe7c94e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql @@ -0,0 +1,60 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp"','U') is not null + begin + drop table test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view" as + +-- Final base SQL model +select + _airbyte_nested_strea__nto_long_names_hashid, + double_array_data, + "DATA", + "column`_''with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_partition_hashid +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab3" +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names" +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp" FROM + "test_normalization".test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_nested_stream_with_co___long_names_partition__dbt_tmp_cci' + AND object_id=object_id('test_normalization_nested_stream_with_co___long_names_partition__dbt_tmp') + ) + DROP index test_normalization.nested_stream_with_co___long_names_partition__dbt_tmp.test_normalization_nested_stream_with_co___long_names_partition__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co___long_names_partition__dbt_tmp_cci + ON test_normalization.nested_stream_with_co___long_names_partition__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql new file mode 100644 index 000000000000..31f9e07c8505 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp"','U') is not null + begin + drop table test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view" as + +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_data_hashid +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab3" +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp" FROM + "test_normalization".test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_nested_stream_with_co___names_partition_data__dbt_tmp_cci' + AND object_id=object_id('test_normalization_nested_stream_with_co___names_partition_data__dbt_tmp') + ) + DROP index test_normalization.nested_stream_with_co___names_partition_data__dbt_tmp.test_normalization_nested_stream_with_co___names_partition_data__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co___names_partition_data__dbt_tmp_cci + ON test_normalization.nested_stream_with_co___names_partition_data__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql new file mode 100644 index 000000000000..81ba784289fc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp"','U') is not null + begin + drop table test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view" as + +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_column___with__quotes_hashid +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab3" +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp" FROM + "test_normalization".test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_nested_stream_with_co__column___with__quotes__dbt_tmp_cci' + AND object_id=object_id('test_normalization_nested_stream_with_co__column___with__quotes__dbt_tmp') + ) + DROP index test_normalization.nested_stream_with_co__column___with__quotes__dbt_tmp.test_normalization_nested_stream_with_co__column___with__quotes__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co__column___with__quotes__dbt_tmp_cci + ON test_normalization.nested_stream_with_co__column___with__quotes__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql new file mode 100644 index 000000000000..1e10857af549 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp"','U') is not null + begin + drop table test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view" as + +-- Final base SQL model +select + _airbyte_partition_hashid, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_double_array_data_hashid +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab3" +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp" FROM + "test_normalization".test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_nested_stream_with_co__ion_double_array_data__dbt_tmp_cci' + AND object_id=object_id('test_normalization_nested_stream_with_co__ion_double_array_data__dbt_tmp') + ) + DROP index test_normalization.nested_stream_with_co__ion_double_array_data__dbt_tmp.test_normalization_nested_stream_with_co__ion_double_array_data__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co__ion_double_array_data__dbt_tmp_cci + ON test_normalization.nested_stream_with_co__ion_double_array_data__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql new file mode 100644 index 000000000000..5facac722a59 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp"','U') is not null + begin + drop table test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view" as + +-- Final base SQL model +select + id, + "date", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_non_nested_s__nto_long_names_hashid +from "test_normalization"._airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab3" +-- non_nested_stream_wit__lting_into_long_names from "test_normalization".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp" FROM + "test_normalization".test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_non_nested_stream_wit__lting_into_long_names__dbt_tmp_cci' + AND object_id=object_id('test_normalization_non_nested_stream_wit__lting_into_long_names__dbt_tmp') + ) + DROP index test_normalization.non_nested_stream_wit__lting_into_long_names__dbt_tmp.test_normalization_non_nested_stream_wit__lting_into_long_names__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_non_nested_stream_wit__lting_into_long_names__dbt_tmp_cci + ON test_normalization.non_nested_stream_wit__lting_into_long_names__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql new file mode 100644 index 000000000000..8c155fdfc51b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -0,0 +1,59 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias_children__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."unnest_alias_children__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias_children__dbt_tmp"','U') is not null + begin + drop table test_normalization."unnest_alias_children__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."unnest_alias_children__dbt_tmp_temp_view" as + +-- Final base SQL model +select + _airbyte_unnest_alias_hashid, + ab_id, + owner, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_children_hashid +from "test_normalization"._airbyte_test_normalization."unnest_alias_children_ab3" +-- children at unnest_alias/children from "test_normalization".test_normalization."unnest_alias" +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."unnest_alias_children__dbt_tmp" FROM + "test_normalization".test_normalization."unnest_alias_children__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias_children__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."unnest_alias_children__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_unnest_alias_children__dbt_tmp_cci' + AND object_id=object_id('test_normalization_unnest_alias_children__dbt_tmp') + ) + DROP index test_normalization.unnest_alias_children__dbt_tmp.test_normalization_unnest_alias_children__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_unnest_alias_children__dbt_tmp_cci + ON test_normalization.unnest_alias_children__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql new file mode 100644 index 000000000000..e3fbbeed2073 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias_children_owner__dbt_tmp"','U') is not null + begin + drop table test_normalization."unnest_alias_children_owner__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view" as + +-- Final base SQL model +select + _airbyte_children_hashid, + owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_owner_hashid +from "test_normalization"._airbyte_test_normalization."unnest_alias_children_owner_ab3" +-- owner at unnest_alias/children/owner from "test_normalization".test_normalization."unnest_alias_children" +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."unnest_alias_children_owner__dbt_tmp" FROM + "test_normalization".test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_unnest_alias_children_owner__dbt_tmp_cci' + AND object_id=object_id('test_normalization_unnest_alias_children_owner__dbt_tmp') + ) + DROP index test_normalization.unnest_alias_children_owner__dbt_tmp.test_normalization_unnest_alias_children_owner__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_unnest_alias_children_owner__dbt_tmp_cci + ON test_normalization.unnest_alias_children_owner__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql new file mode 100644 index 000000000000..b6091947b13a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp"','U') is not null + begin + drop table test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view" as + +-- Final base SQL model +select + id, + "date", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_simple_strea__nto_long_names_hashid +from "test_normalization"._airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab3" +-- simple_stream_with_na__lting_into_long_names from "test_normalization".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp" FROM + "test_normalization".test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_namespace_simple_stream_with_na__lting_into_long_names__dbt_tmp_cci' + AND object_id=object_id('test_normalization_namespace_simple_stream_with_na__lting_into_long_names__dbt_tmp') + ) + DROP index test_normalization_namespace.simple_stream_with_na__lting_into_long_names__dbt_tmp.test_normalization_namespace_simple_stream_with_na__lting_into_long_names__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_namespace_simple_stream_with_na__lting_into_long_names__dbt_tmp_cci + ON test_normalization_namespace.simple_stream_with_na__lting_into_long_names__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql new file mode 100644 index 000000000000..871fcf869df0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql @@ -0,0 +1,21 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as + VARCHAR(max)), ''''), ''-'', coalesce(cast("partition" as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_nested_strea__nto_long_names_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab2" tmp +-- nested_stream_with_co__lting_into_long_names +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql deleted file mode 100644 index a271eb33daa9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql +++ /dev/null @@ -1,19 +0,0 @@ -with table_row_counts as ( - select distinct count(*) as row_count, 2 as expected_count - from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} -union all - select distinct count(*) as row_count, 2 as expected_count - from {{ ref('nested_stream_with_co__lting_into_long_names') }} -union all - select distinct count(*) as row_count, 2 as expected_count - from {{ ref('nested_stream_with_co___long_names_partition') }} -union all - select count(distinct currency) as row_count, 1 as expected_count - from {{ ref('nested_stream_with_co___names_partition_data') }} --- union all --- select count(distinct id) as row_count, 3 as expected_count --- from {{ ref('nested_stream_with_co__ion_double_array_data') }} -) -select * -from table_row_counts -where row_count != expected_count diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql index d3df56ab3d35..a256ac12c37a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_array('_airbyte_data', ['conflict_stream_array'], ['conflict_stream_array']) }} as conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} as table_alias -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql index 2f0da1bb06e7..e770c48bd904 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_array_ab1') }} -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql index 0a92c655f92a..4c6cb6138a52 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('conflict_stream_array_ab2') }} tmp -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql index 77e7bd31c512..aca9d0993391 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql @@ -1,10 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_conflict_stream_name_2_hashid, {{ json_extract_scalar('conflict_stream_name', ['groups'], ['groups']) }} as groups, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_conflict_stream_name') }} as table_alias -where conflict_stream_name is not null -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql index ebd4c64d6b0b..e7ef46dec94e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_conflict_stream_name_2_hashid, cast(groups as {{ dbt_utils.type_string() }}) as groups, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name____conflict_stream_name_ab1') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql index 09eaffe2cf54..839344888f3b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('conflict_stream_name____conflict_stream_name_ab2') }} tmp -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql index 8556d7f50afb..9128769dd58b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract('table_alias', '_airbyte_data', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} as table_alias -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql index a1d77f379bce..7612525c0943 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_ab1') }} -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql index dc562af89ba1..b9d505d0445a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('conflict_stream_name_ab2') }} tmp -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql index d6e2edb75ced..97ea3e982cbe 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql @@ -1,10 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_conflict_stream_name_hashid, {{ json_extract('table_alias', 'conflict_stream_name', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name') }} as table_alias -where conflict_stream_name is not null -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql index 463a878933f8..80799bfca4cf 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_conflict_stream_name_hashid, cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_conflict_stream_name_ab1') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql index f8d96123ce25..1cc503d67e41 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('conflict_stream_name_conflict_stream_name_ab2') }} tmp -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql index be57970e399e..6169df7dc10b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['conflict_stream_scalar'], ['conflict_stream_scalar']) }} as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} as table_alias -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql index cf0d9deb51cd..51d157d48e14 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast(conflict_stream_scalar as {{ dbt_utils.type_bigint() }}) as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_scalar_ab1') }} -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql index 781c4bbfc391..f71e7c572052 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('conflict_stream_scalar_ab2') }} tmp -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql index 04ef717d09bf..a0784863bc52 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql @@ -1,12 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_nested_strea__nto_long_names_hashid, {{ json_extract_array(adapter.quote('partition'), ['double_array_data'], ['double_array_data']) }} as double_array_data, {{ json_extract_array(adapter.quote('partition'), ['DATA'], ['DATA']) }} as {{ adapter.quote('DATA') }}, {{ json_extract_array(adapter.quote('partition'), ['column`_\'with"_quotes'], ['column___with__quotes']) }} as {{ adapter.quote('column`_\'with""_quotes') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co__lting_into_long_names') }} as table_alias -where {{ adapter.quote('partition') }} is not null -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and {{ adapter.quote('partition') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql index 37d5abd5baff..aafc5126e7d6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql @@ -1,11 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_nested_strea__nto_long_names_hashid, double_array_data, {{ adapter.quote('DATA') }}, {{ adapter.quote('column`_\'with""_quotes') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co___long_names_partition_ab1') }} -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql index c9befa862c0a..a0546907dec4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -10,4 +14,5 @@ select tmp.* from {{ ref('nested_stream_with_co___long_names_partition_ab2') }} tmp -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql index d5c4708f7e07..46f5b7c11ae2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql @@ -1,12 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_co___long_names_partition', 'partition', adapter.quote('DATA')) }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value(adapter.quote('DATA')), ['currency'], ['currency']) }} as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co___long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', adapter.quote('DATA')) }} -where {{ adapter.quote('DATA') }} is not null -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +{{ cross_join_unnest('partition', adapter.quote('DATA')) }} +where 1 = 1 +and {{ adapter.quote('DATA') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql index bdf11a7ed223..8c35fe50450c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(currency as {{ dbt_utils.type_string() }}) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co___names_partition_data_ab1') }} -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql index 11cdb80699c0..a841db507bae 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('nested_stream_with_co___names_partition_data_ab2') }} tmp -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql index fa16aeba6633..8cd21390af91 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql @@ -1,12 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_co___long_names_partition', 'partition', adapter.quote('column`_\'with""_quotes')) }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value(adapter.quote('column`_\'with""_quotes')), ['currency'], ['currency']) }} as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co___long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', adapter.quote('column`_\'with""_quotes')) }} -where {{ adapter.quote('column`_\'with""_quotes') }} is not null -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +{{ cross_join_unnest('partition', adapter.quote('column`_\'with""_quotes')) }} +where 1 = 1 +and {{ adapter.quote('column`_\'with""_quotes') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql index 2f4622e3028a..151455db8db0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(currency as {{ dbt_utils.type_string() }}) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co__column___with__quotes_ab1') }} -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql index 62ba655e2e45..fca643924c08 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('nested_stream_with_co__column___with__quotes_ab2') }} tmp -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql index 5b3a0601175b..a191b72022c5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql @@ -1,12 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_co___long_names_partition', 'partition', 'double_array_data') }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value('double_array_data'), ['id'], ['id']) }} as id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co___long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', 'double_array_data') }} -where double_array_data is not null -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +{{ cross_join_unnest('partition', 'double_array_data') }} +where 1 = 1 +and double_array_data is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql index e46023b76c26..517f1b6759ba 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(id as {{ dbt_utils.type_string() }}) as id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co__ion_double_array_data_ab1') }} -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql index 7268452cad18..5eee9d197132 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('nested_stream_with_co__ion_double_array_data_ab2') }} tmp -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql index e37aca6c48d3..3b66fee00d27 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql @@ -1,10 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, {{ json_extract('table_alias', '_airbyte_data', ['partition'], ['partition']) }} as {{ adapter.quote('partition') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} as table_alias -- nested_stream_with_co__lting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql index 68218a656935..fa82dea1c04a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql @@ -1,10 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast({{ adapter.quote('date') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('date') }}, cast({{ adapter.quote('partition') }} as {{ type_json() }}) as {{ adapter.quote('partition') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co__lting_into_long_names_ab1') }} -- nested_stream_with_co__lting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql deleted file mode 100644 index 0c9056f13886..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql +++ /dev/null @@ -1,12 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - adapter.quote('date'), - adapter.quote('partition'), - ]) }} as _airbyte_nested_strea__nto_long_names_hashid, - tmp.* -from {{ ref('nested_stream_with_co__lting_into_long_names_ab2') }} tmp --- nested_stream_with_co__lting_into_long_names - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab4.sql deleted file mode 100644 index ac961f2763ba..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_nested_strea__nto_long_names_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('nested_stream_with_co__lting_into_long_names_ab3') }} tmp --- nested_stream_with_co__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql index 66f8e4943db1..4214cc379780 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names') }} as table_alias -- non_nested_stream_wit__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql index ee6798c91bc0..d68cc555b13e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast({{ adapter.quote('date') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('date') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('non_nested_stream_wit__lting_into_long_names_ab1') }} -- non_nested_stream_wit__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql index 5bdd8ae4c448..b7f3b08618eb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('non_nested_stream_wit__lting_into_long_names_ab2') }} tmp -- non_nested_stream_wit__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql index 15b34c1ca7d4..a7877e1d2dde 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_array('_airbyte_data', ['children'], ['children']) }} as children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} as table_alias -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql index a416538b5422..b397e46d7b04 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_ab1') }} -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql index 0c6703933b10..9f2bd6c9d3af 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,6 @@ select tmp.* from {{ ref('unnest_alias_ab2') }} tmp -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql index 57abeb23159f..7d715421a2ab 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql @@ -1,13 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('unnest_alias', 'unnest_alias', 'children') }} select _airbyte_unnest_alias_hashid, {{ json_extract_scalar(unnested_column_value('children'), ['ab_id'], ['ab_id']) }} as ab_id, {{ json_extract('', unnested_column_value('children'), ['owner'], ['owner']) }} as owner, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias') }} as table_alias -{{ cross_join_unnest('unnest_alias', 'children') }} -where children is not null -- children at unnest_alias/children +{{ cross_join_unnest('unnest_alias', 'children') }} +where 1 = 1 +and children is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql index 8db853c1d601..cfdd1e1a68e9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql @@ -1,10 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_unnest_alias_hashid, cast(ab_id as {{ dbt_utils.type_bigint() }}) as ab_id, cast(owner as {{ type_json() }}) as owner, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children_ab1') }} -- children at unnest_alias/children +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql index 9d84f3be6aa0..eb45635c9b9a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -9,4 +13,5 @@ select tmp.* from {{ ref('unnest_alias_children_ab2') }} tmp -- children at unnest_alias/children +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql index 4f28c4f8b5cb..3c4f63bb121b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql @@ -1,10 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_children_hashid, {{ json_extract_scalar('owner', ['owner_id'], ['owner_id']) }} as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children') }} as table_alias -where owner is not null -- owner at unnest_alias/children/owner +where 1 = 1 +and owner is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql index a0f474c62db7..109a7652533a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_children_hashid, cast(owner_id as {{ dbt_utils.type_bigint() }}) as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children_owner_ab1') }} -- owner at unnest_alias/children/owner +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql index c5f6562c34c2..53a716a401c1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('unnest_alias_children_owner_ab2') }} tmp -- owner at unnest_alias/children/owner +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql index 1afa8d54d15e..52601ad2201a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization_namespace', '_airbyte_raw_simple_stream_with_namespace_resulting_into_long_names') }} as table_alias -- simple_stream_with_na__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql index 185b0e2f8a39..28487a3d51cd 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast({{ adapter.quote('date') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('date') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('simple_stream_with_na__lting_into_long_names_ab1') }} -- simple_stream_with_na__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql index 5766cc587575..0e264457735e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('simple_stream_with_na__lting_into_long_names_ab2') }} tmp -- simple_stream_with_na__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql new file mode 100644 index 000000000000..b9817bc561a3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql @@ -0,0 +1,107 @@ +{{ config( + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('nested_stream_with_co__lting_into_long_names_ab3') }} + -- nested_stream_with_co__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('nested_stream_with_co__lting_into_long_names_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('nested_stream_with_co__lting_into_long_names_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('nested_stream_with_co__lting_into_long_names_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('nested_stream_with_co__lting_into_long_names_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('nested_stream_with_co__lting_into_long_names_ab3') }} + -- nested_stream_with_co__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + {{ adapter.quote('date') }}, + {{ adapter.quote('partition') }}, + {{ adapter.quote('date') }} as _airbyte_start_at, + lag({{ adapter.quote('date') }}) over ( + partition by id + order by + {{ adapter.quote('date') }} desc, + {{ adapter.quote('date') }} desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag({{ adapter.quote('date') }}) over ( + partition by id + order by + {{ adapter.quote('date') }} desc, + {{ adapter.quote('date') }} desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_nested_strea__nto_long_names_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + {{ adapter.quote('date') }}, + {{ adapter.quote('partition') }}, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_nested_strea__nto_long_names_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql new file mode 100644 index 000000000000..16bc6c540153 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql @@ -0,0 +1,21 @@ +{{ config( + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + id, + {{ adapter.quote('date') }}, + {{ adapter.quote('partition') }}, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_nested_strea__nto_long_names_hashid +from {{ ref('nested_stream_with_co__lting_into_long_names_scd') }} +-- nested_stream_with_co__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..dad253380255 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql @@ -0,0 +1,18 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + id, + children, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_unnest_alias_hashid +from {{ ref('unnest_alias_ab3') }} +-- unnest_alias from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql index 9ffda6086e35..6e616cb068a4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, conflict_stream_array, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_array_hashid from {{ ref('conflict_stream_array_ab3') }} -- conflict_stream_array from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql index 8a83ccc01e31..6b70e1601f56 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_hashid from {{ ref('conflict_stream_name_ab3') }} -- conflict_stream_name from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql index bc23350b4f48..5244eb2cb066 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_conflict_stream_name_2_hashid, groups, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_3_hashid from {{ ref('conflict_stream_name____conflict_stream_name_ab3') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name_conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql index 94a9dc3d0139..d309f67ec032 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_conflict_stream_name_hashid, conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_2_hashid from {{ ref('conflict_stream_name_conflict_stream_name_ab3') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql index 9d7acc846fd0..3e196df09e88 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, conflict_stream_scalar, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_scalar_hashid from {{ ref('conflict_stream_scalar_ab3') }} -- conflict_stream_scalar from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql index 463ee8735a5d..223dd426b23f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql @@ -1,12 +1,19 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_nested_strea__nto_long_names_hashid, double_array_data, {{ adapter.quote('DATA') }}, {{ adapter.quote('column`_\'with""_quotes') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_partition_hashid from {{ ref('nested_stream_with_co___long_names_partition_ab3') }} -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from {{ ref('nested_stream_with_co__lting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql index 9ab923f69627..e9f3fe55ca77 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_data_hashid from {{ ref('nested_stream_with_co___names_partition_data_ab3') }} -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from {{ ref('nested_stream_with_co___long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql index 8b126445559b..41147ab78466 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_column___with__quotes_hashid from {{ ref('nested_stream_with_co__column___with__quotes_ab3') }} -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from {{ ref('nested_stream_with_co___long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql index d36f1659ee95..0ef5f8b3ad14 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, id, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_double_array_data_hashid from {{ ref('nested_stream_with_co__ion_double_array_data_ab3') }} -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from {{ ref('nested_stream_with_co___long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql deleted file mode 100644 index 28635caec9e9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql +++ /dev/null @@ -1,12 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - {{ adapter.quote('date') }}, - {{ adapter.quote('partition') }}, - _airbyte_emitted_at, - _airbyte_nested_strea__nto_long_names_hashid -from {{ ref('nested_stream_with_co__lting_into_long_names_scd') }} --- nested_stream_with_co__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} -where _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql deleted file mode 100644 index c3ed2b7aad48..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - {{ adapter.quote('date') }}, - {{ adapter.quote('partition') }}, - {{ adapter.quote('date') }} as _airbyte_start_at, - lag({{ adapter.quote('date') }}) over ( - partition by id - order by {{ adapter.quote('date') }} desc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag({{ adapter.quote('date') }}) over ( - partition by id - order by {{ adapter.quote('date') }} desc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_nested_strea__nto_long_names_hashid -from {{ ref('nested_stream_with_co__lting_into_long_names_ab4') }} --- nested_stream_with_co__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql index 0cb2abc14b0e..ac7044df4e60 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, {{ adapter.quote('date') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_non_nested_s__nto_long_names_hashid from {{ ref('non_nested_stream_wit__lting_into_long_names_ab3') }} -- non_nested_stream_wit__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql deleted file mode 100644 index 7a570c3836cb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,10 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - children, - _airbyte_emitted_at, - _airbyte_unnest_alias_hashid -from {{ ref('unnest_alias_ab3') }} --- unnest_alias from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql index 77b2b0ad854e..aaeaad626502 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -1,11 +1,18 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_unnest_alias_hashid, ab_id, owner, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_children_hashid from {{ ref('unnest_alias_children_ab3') }} -- children at unnest_alias/children from {{ ref('unnest_alias') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql index 0ccf8e33d8d6..e6710c9eb232 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_children_hashid, owner_id, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_owner_hashid from {{ ref('unnest_alias_children_owner_ab3') }} -- owner at unnest_alias/children/owner from {{ ref('unnest_alias_children') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql index 11a61e98ee75..a42310343b41 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization_namespace", tags=["top-level"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization_namespace", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, {{ adapter.quote('date') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_simple_strea__nto_long_names_hashid from {{ ref('simple_stream_with_na__lting_into_long_names_ab3') }} -- simple_stream_with_na__lting_into_long_names from {{ source('test_normalization_namespace', '_airbyte_raw_simple_stream_with_namespace_resulting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql new file mode 100644 index 000000000000..58cec3241028 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql @@ -0,0 +1,18 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + adapter.quote('date'), + adapter.quote('partition'), + ]) }} as _airbyte_nested_strea__nto_long_names_hashid, + tmp.* +from {{ ref('nested_stream_with_co__lting_into_long_names_ab2') }} tmp +-- nested_stream_with_co__lting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql new file mode 100644 index 000000000000..1567f2b78797 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql @@ -0,0 +1,15 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_array_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_query(_airbyte_data, ''$."conflict_stream_array"'') as conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_conflict_stream_array as table_alias +-- conflict_stream_array +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql new file mode 100644 index 000000000000..a16d74e97ea4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_array_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + VARCHAR(max)) as id, + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."conflict_stream_array_ab1" +-- conflict_stream_array +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql new file mode 100644 index 000000000000..e5bde2f88eab --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql @@ -0,0 +1,20 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_array_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(cast(conflict_stream_array as + VARCHAR(max)) as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_array_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."conflict_stream_array_ab2" tmp +-- conflict_stream_array +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..711edc45508d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_2_hashid, + json_value(conflict_stream_name, ''$."groups"'') as groups, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name" as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..a8f6a33ad157 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_2_hashid, + cast(groups as + VARCHAR(max)) as groups, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab1" +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..4c510332a6ee --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(groups as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_name_3_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab2" tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..8f1fc7fe16db --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql @@ -0,0 +1,15 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_name_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_query(_airbyte_data, ''$."conflict_stream_name"'') as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_conflict_stream_name as table_alias +-- conflict_stream_name +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..1e106f88c407 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_name_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + VARCHAR(max)) as id, + cast(conflict_stream_name as VARCHAR(max)) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."conflict_stream_name_ab1" +-- conflict_stream_name +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..dd136bf9bad9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_name_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(conflict_stream_name as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_name_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."conflict_stream_name_ab2" tmp +-- conflict_stream_name +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..26744bb811ae --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_hashid, + json_query(conflict_stream_name, ''$."conflict_stream_name"'') as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization."conflict_stream_name" as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..6a630d1e24c1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql @@ -0,0 +1,15 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_hashid, + cast(conflict_stream_name as VARCHAR(max)) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab1" +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..cbe3f02a60f7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_conflict_stream_name_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(conflict_stream_name as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_name_2_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab2" tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql new file mode 100644 index 000000000000..ecce4bc0d160 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql @@ -0,0 +1,15 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_scalar_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."conflict_stream_scalar"'') as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_conflict_stream_scalar as table_alias +-- conflict_stream_scalar +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql new file mode 100644 index 000000000000..0cb7a859994b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql @@ -0,0 +1,18 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_scalar_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + VARCHAR(max)) as id, + cast(conflict_stream_scalar as + bigint +) as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."conflict_stream_scalar_ab1" +-- conflict_stream_scalar +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql new file mode 100644 index 000000000000..54f208622b20 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_scalar_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(conflict_stream_scalar as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_scalar_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."conflict_stream_scalar_ab2" tmp +-- conflict_stream_scalar +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql new file mode 100644 index 000000000000..d74cd1c7389f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql @@ -0,0 +1,18 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_nested_strea__nto_long_names_hashid, + json_query("partition", ''$."double_array_data"'') as double_array_data, + json_query("partition", ''$."DATA"'') as "DATA", + json_query("partition", ''$."column`_''''with\"_quotes"'') as "column`_''with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names" as table_alias +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and "partition" is not null + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql new file mode 100644 index 000000000000..90c0eafb817d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql @@ -0,0 +1,17 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_nested_strea__nto_long_names_hashid, + double_array_data, + "DATA", + "column`_''with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab1" +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql new file mode 100644 index 000000000000..f898a92e06da --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql @@ -0,0 +1,24 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_nested_strea__nto_long_names_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(cast(double_array_data as + VARCHAR(max)) as + VARCHAR(max)), ''''), ''-'', coalesce(cast(cast("DATA" as + VARCHAR(max)) as + VARCHAR(max)), ''''), ''-'', coalesce(cast(cast("column`_''with""_quotes" as + VARCHAR(max)) as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_partition_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab2" tmp +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql new file mode 100644 index 000000000000..deb088fe4931 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql @@ -0,0 +1,26 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + json_value( + "DATA".value, ''$."currency"'') as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" as table_alias +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA + + CROSS APPLY ( + SELECT [value] = CASE + WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) + WHEN [type] = 5 THEN [value] + END + FROM OPENJSON("DATA") + ) AS "DATA" +where 1 = 1 +and "DATA" is not null + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql new file mode 100644 index 000000000000..505eb01f4d31 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as + VARCHAR(max)) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab1" +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql new file mode 100644 index 000000000000..89631a486695 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_partition_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_data_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab2" tmp +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql new file mode 100644 index 000000000000..96a6f2a9eacc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql @@ -0,0 +1,26 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + json_value( + "column`_''with""_quotes".value, ''$."currency"'') as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" as table_alias +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes + + CROSS APPLY ( + SELECT [value] = CASE + WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) + WHEN [type] = 5 THEN [value] + END + FROM OPENJSON("column`_''with""_quotes") + ) AS "column`_''with""_quotes" +where 1 = 1 +and "column`_''with""_quotes" is not null + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql new file mode 100644 index 000000000000..9edfb5ade031 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as + VARCHAR(max)) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab1" +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql new file mode 100644 index 000000000000..1da3ab107876 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_partition_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_column___with__quotes_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab2" tmp +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql new file mode 100644 index 000000000000..7e83428589ca --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql @@ -0,0 +1,26 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + json_value( + double_array_data.value, ''$."id"'') as id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" as table_alias +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data + + CROSS APPLY ( + SELECT [value] = CASE + WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) + WHEN [type] = 5 THEN [value] + END + FROM OPENJSON(double_array_data) + ) AS double_array_data +where 1 = 1 +and double_array_data is not null + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql new file mode 100644 index 000000000000..1eb3236e85d9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(id as + VARCHAR(max)) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab1" +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql new file mode 100644 index 000000000000..b6e73981f3fb --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_partition_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(id as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_double_array_data_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab2" tmp +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql new file mode 100644 index 000000000000..e921a062d3b7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql @@ -0,0 +1,17 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."date"'') as "date", + json_query(_airbyte_data, ''$."partition"'') as "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias +-- nested_stream_with_co__lting_into_long_names +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql new file mode 100644 index 000000000000..5eafd0d71a6b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + VARCHAR(max)) as id, + cast("date" as + VARCHAR(max)) as "date", + cast("partition" as VARCHAR(max)) as "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab1" +-- nested_stream_with_co__lting_into_long_names +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql new file mode 100644 index 000000000000..c81d35772445 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql @@ -0,0 +1,15 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."date"'') as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias +-- non_nested_stream_wit__lting_into_long_names +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql new file mode 100644 index 000000000000..4abb31531a75 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql @@ -0,0 +1,17 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + VARCHAR(max)) as id, + cast("date" as + VARCHAR(max)) as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab1" +-- non_nested_stream_wit__lting_into_long_names +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql new file mode 100644 index 000000000000..094bd24079d4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_non_nested_s__nto_long_names_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab2" tmp +-- non_nested_stream_wit__lting_into_long_names +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql new file mode 100644 index 000000000000..177d8bda7d07 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."unnest_alias_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_query(_airbyte_data, ''$."children"'') as children, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_unnest_alias as table_alias +-- unnest_alias +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql new file mode 100644 index 000000000000..a9ef564eff36 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql @@ -0,0 +1,18 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."unnest_alias_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + children, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."unnest_alias_ab1" +-- unnest_alias +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql new file mode 100644 index 000000000000..4544841f480f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql @@ -0,0 +1,21 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."unnest_alias_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(cast(children as + VARCHAR(max)) as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_unnest_alias_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."unnest_alias_ab2" tmp +-- unnest_alias +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql new file mode 100644 index 000000000000..d9f07f78b06c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql @@ -0,0 +1,28 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."unnest_alias_children_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_unnest_alias_hashid, + json_value( + children.value, ''$."ab_id"'') as ab_id, + json_query( + children.value, ''$."owner"'') as owner, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization."unnest_alias" as table_alias +-- children at unnest_alias/children + + CROSS APPLY ( + SELECT [value] = CASE + WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) + WHEN [type] = 5 THEN [value] + END + FROM OPENJSON(children) + ) AS children +where 1 = 1 +and children is not null + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql new file mode 100644 index 000000000000..c8d4bf47083c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql @@ -0,0 +1,18 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."unnest_alias_children_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_unnest_alias_hashid, + cast(ab_id as + bigint +) as ab_id, + cast(owner as VARCHAR(max)) as owner, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."unnest_alias_children_ab1" +-- children at unnest_alias/children +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql new file mode 100644 index 000000000000..6ad3a1c4caf8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql @@ -0,0 +1,20 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."unnest_alias_children_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_unnest_alias_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(ab_id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(owner as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_children_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."unnest_alias_children_ab2" tmp +-- children at unnest_alias/children +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql new file mode 100644 index 000000000000..7f49a4c7f4ea --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."unnest_alias_children_owner_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_children_hashid, + json_value(owner, ''$."owner_id"'') as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization."unnest_alias_children" as table_alias +-- owner at unnest_alias/children/owner +where 1 = 1 +and owner is not null + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql new file mode 100644 index 000000000000..9cf0dca5efb6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql @@ -0,0 +1,17 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."unnest_alias_children_owner_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_children_hashid, + cast(owner_id as + bigint +) as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."unnest_alias_children_owner_ab1" +-- owner at unnest_alias/children/owner +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql new file mode 100644 index 000000000000..fe2cfe2e2a98 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."unnest_alias_children_owner_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_children_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(owner_id as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_owner_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."unnest_alias_children_owner_ab2" tmp +-- owner at unnest_alias/children/owner +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql new file mode 100644 index 000000000000..a86efc7f2153 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql @@ -0,0 +1,15 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."date"'') as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias +-- simple_stream_with_na__lting_into_long_names +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql new file mode 100644 index 000000000000..077cbe05b163 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql @@ -0,0 +1,17 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + VARCHAR(max)) as id, + cast("date" as + VARCHAR(max)) as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab1" +-- simple_stream_with_na__lting_into_long_names +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql new file mode 100644 index 000000000000..f2f9167067b8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_simple_strea__nto_long_names_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab2" tmp +-- simple_stream_with_na__lting_into_long_names +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql new file mode 100644 index 000000000000..9089838aa50f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql @@ -0,0 +1,14 @@ + + delete + from "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "test_normalization".test_normalization."#nested_stream_with_co__lting_into_long_names_scd__dbt_tmp" + ); + + insert into "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "date", "partition", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_strea__nto_long_names_hashid") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "date", "partition", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_strea__nto_long_names_hashid" + from "test_normalization".test_normalization."#nested_stream_with_co__lting_into_long_names_scd__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql new file mode 100644 index 000000000000..ea2ee46a0293 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql @@ -0,0 +1,14 @@ + + delete + from "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "test_normalization".test_normalization."#nested_stream_with_co__lting_into_long_names__dbt_tmp" + ); + + insert into "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names" ("_airbyte_unique_key", "id", "date", "partition", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_strea__nto_long_names_hashid") + ( + select "_airbyte_unique_key", "id", "date", "partition", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_strea__nto_long_names_hashid" + from "test_normalization".test_normalization."#nested_stream_with_co__lting_into_long_names__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..cbef7f6209d1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql @@ -0,0 +1,14 @@ + + delete + from "test_normalization".test_normalization."unnest_alias" + where (_airbyte_ab_id) in ( + select (_airbyte_ab_id) + from "test_normalization".test_normalization."#unnest_alias__dbt_tmp" + ); + + insert into "test_normalization".test_normalization."unnest_alias" ("id", "children", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_unnest_alias_hashid") + ( + select "id", "children", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_unnest_alias_hashid" + from "test_normalization".test_normalization."#unnest_alias__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql new file mode 100644 index 000000000000..2ffefaacaa27 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_array__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_array__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_array__dbt_tmp"','U') is not null + begin + drop table test_normalization."conflict_stream_array__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."conflict_stream_array__dbt_tmp_temp_view" as + +-- Final base SQL model +select + id, + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_conflict_stream_array_hashid +from "test_normalization"._airbyte_test_normalization."conflict_stream_array_ab3" +-- conflict_stream_array from "test_normalization".test_normalization._airbyte_raw_conflict_stream_array +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."conflict_stream_array__dbt_tmp" FROM + "test_normalization".test_normalization."conflict_stream_array__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_array__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_array__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_conflict_stream_array__dbt_tmp_cci' + AND object_id=object_id('test_normalization_conflict_stream_array__dbt_tmp') + ) + DROP index test_normalization.conflict_stream_array__dbt_tmp.test_normalization_conflict_stream_array__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_array__dbt_tmp_cci + ON test_normalization.conflict_stream_array__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql new file mode 100644 index 000000000000..e2c0ff6666d5 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_name__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name__dbt_tmp"','U') is not null + begin + drop table test_normalization."conflict_stream_name__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."conflict_stream_name__dbt_tmp_temp_view" as + +-- Final base SQL model +select + id, + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_hashid +from "test_normalization"._airbyte_test_normalization."conflict_stream_name_ab3" +-- conflict_stream_name from "test_normalization".test_normalization._airbyte_raw_conflict_stream_name +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."conflict_stream_name__dbt_tmp" FROM + "test_normalization".test_normalization."conflict_stream_name__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_name__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_conflict_stream_name__dbt_tmp_cci' + AND object_id=object_id('test_normalization_conflict_stream_name__dbt_tmp') + ) + DROP index test_normalization.conflict_stream_name__dbt_tmp.test_normalization_conflict_stream_name__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_name__dbt_tmp_cci + ON test_normalization.conflict_stream_name__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql new file mode 100644 index 000000000000..cc5f9ded53f2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp"','U') is not null + begin + drop table test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view" as + +-- Final base SQL model +select + _airbyte_conflict_stream_name_2_hashid, + groups, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_3_hashid +from "test_normalization"._airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab3" +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name" +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp" FROM + "test_normalization".test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_conflict_stream_name____conflict_stream_name__dbt_tmp_cci' + AND object_id=object_id('test_normalization_conflict_stream_name____conflict_stream_name__dbt_tmp') + ) + DROP index test_normalization.conflict_stream_name____conflict_stream_name__dbt_tmp.test_normalization_conflict_stream_name____conflict_stream_name__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_name____conflict_stream_name__dbt_tmp_cci + ON test_normalization.conflict_stream_name____conflict_stream_name__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql new file mode 100644 index 000000000000..79de98cb1242 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp"','U') is not null + begin + drop table test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view" as + +-- Final base SQL model +select + _airbyte_conflict_stream_name_hashid, + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_2_hashid +from "test_normalization"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab3" +-- conflict_stream_name at conflict_stream_name/conflict_stream_name from "test_normalization".test_normalization."conflict_stream_name" +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp" FROM + "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_conflict_stream_name_conflict_stream_name__dbt_tmp_cci' + AND object_id=object_id('test_normalization_conflict_stream_name_conflict_stream_name__dbt_tmp') + ) + DROP index test_normalization.conflict_stream_name_conflict_stream_name__dbt_tmp.test_normalization_conflict_stream_name_conflict_stream_name__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_name_conflict_stream_name__dbt_tmp_cci + ON test_normalization.conflict_stream_name_conflict_stream_name__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql new file mode 100644 index 000000000000..62ee6566a04a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_scalar__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_scalar__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_scalar__dbt_tmp"','U') is not null + begin + drop table test_normalization."conflict_stream_scalar__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."conflict_stream_scalar__dbt_tmp_temp_view" as + +-- Final base SQL model +select + id, + conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_conflict_stream_scalar_hashid +from "test_normalization"._airbyte_test_normalization."conflict_stream_scalar_ab3" +-- conflict_stream_scalar from "test_normalization".test_normalization._airbyte_raw_conflict_stream_scalar +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."conflict_stream_scalar__dbt_tmp" FROM + "test_normalization".test_normalization."conflict_stream_scalar__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_scalar__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_scalar__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_conflict_stream_scalar__dbt_tmp_cci' + AND object_id=object_id('test_normalization_conflict_stream_scalar__dbt_tmp') + ) + DROP index test_normalization.conflict_stream_scalar__dbt_tmp.test_normalization_conflict_stream_scalar__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_scalar__dbt_tmp_cci + ON test_normalization.conflict_stream_scalar__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql new file mode 100644 index 000000000000..a1277fe7c94e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql @@ -0,0 +1,60 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp"','U') is not null + begin + drop table test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view" as + +-- Final base SQL model +select + _airbyte_nested_strea__nto_long_names_hashid, + double_array_data, + "DATA", + "column`_''with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_partition_hashid +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab3" +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names" +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp" FROM + "test_normalization".test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_nested_stream_with_co___long_names_partition__dbt_tmp_cci' + AND object_id=object_id('test_normalization_nested_stream_with_co___long_names_partition__dbt_tmp') + ) + DROP index test_normalization.nested_stream_with_co___long_names_partition__dbt_tmp.test_normalization_nested_stream_with_co___long_names_partition__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co___long_names_partition__dbt_tmp_cci + ON test_normalization.nested_stream_with_co___long_names_partition__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql new file mode 100644 index 000000000000..31f9e07c8505 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp"','U') is not null + begin + drop table test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view" as + +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_data_hashid +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab3" +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp" FROM + "test_normalization".test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_nested_stream_with_co___names_partition_data__dbt_tmp_cci' + AND object_id=object_id('test_normalization_nested_stream_with_co___names_partition_data__dbt_tmp') + ) + DROP index test_normalization.nested_stream_with_co___names_partition_data__dbt_tmp.test_normalization_nested_stream_with_co___names_partition_data__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co___names_partition_data__dbt_tmp_cci + ON test_normalization.nested_stream_with_co___names_partition_data__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql new file mode 100644 index 000000000000..81ba784289fc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp"','U') is not null + begin + drop table test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view" as + +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_column___with__quotes_hashid +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab3" +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp" FROM + "test_normalization".test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_nested_stream_with_co__column___with__quotes__dbt_tmp_cci' + AND object_id=object_id('test_normalization_nested_stream_with_co__column___with__quotes__dbt_tmp') + ) + DROP index test_normalization.nested_stream_with_co__column___with__quotes__dbt_tmp.test_normalization_nested_stream_with_co__column___with__quotes__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co__column___with__quotes__dbt_tmp_cci + ON test_normalization.nested_stream_with_co__column___with__quotes__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql new file mode 100644 index 000000000000..1e10857af549 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp"','U') is not null + begin + drop table test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view" as + +-- Final base SQL model +select + _airbyte_partition_hashid, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_double_array_data_hashid +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab3" +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp" FROM + "test_normalization".test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_nested_stream_with_co__ion_double_array_data__dbt_tmp_cci' + AND object_id=object_id('test_normalization_nested_stream_with_co__ion_double_array_data__dbt_tmp') + ) + DROP index test_normalization.nested_stream_with_co__ion_double_array_data__dbt_tmp.test_normalization_nested_stream_with_co__ion_double_array_data__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co__ion_double_array_data__dbt_tmp_cci + ON test_normalization.nested_stream_with_co__ion_double_array_data__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql new file mode 100644 index 000000000000..5facac722a59 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp"','U') is not null + begin + drop table test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view" as + +-- Final base SQL model +select + id, + "date", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_non_nested_s__nto_long_names_hashid +from "test_normalization"._airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab3" +-- non_nested_stream_wit__lting_into_long_names from "test_normalization".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp" FROM + "test_normalization".test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_non_nested_stream_wit__lting_into_long_names__dbt_tmp_cci' + AND object_id=object_id('test_normalization_non_nested_stream_wit__lting_into_long_names__dbt_tmp') + ) + DROP index test_normalization.non_nested_stream_wit__lting_into_long_names__dbt_tmp.test_normalization_non_nested_stream_wit__lting_into_long_names__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_non_nested_stream_wit__lting_into_long_names__dbt_tmp_cci + ON test_normalization.non_nested_stream_wit__lting_into_long_names__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql new file mode 100644 index 000000000000..8c155fdfc51b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -0,0 +1,59 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias_children__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."unnest_alias_children__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias_children__dbt_tmp"','U') is not null + begin + drop table test_normalization."unnest_alias_children__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."unnest_alias_children__dbt_tmp_temp_view" as + +-- Final base SQL model +select + _airbyte_unnest_alias_hashid, + ab_id, + owner, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_children_hashid +from "test_normalization"._airbyte_test_normalization."unnest_alias_children_ab3" +-- children at unnest_alias/children from "test_normalization".test_normalization."unnest_alias" +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."unnest_alias_children__dbt_tmp" FROM + "test_normalization".test_normalization."unnest_alias_children__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias_children__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."unnest_alias_children__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_unnest_alias_children__dbt_tmp_cci' + AND object_id=object_id('test_normalization_unnest_alias_children__dbt_tmp') + ) + DROP index test_normalization.unnest_alias_children__dbt_tmp.test_normalization_unnest_alias_children__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_unnest_alias_children__dbt_tmp_cci + ON test_normalization.unnest_alias_children__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql new file mode 100644 index 000000000000..e3fbbeed2073 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias_children_owner__dbt_tmp"','U') is not null + begin + drop table test_normalization."unnest_alias_children_owner__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view" as + +-- Final base SQL model +select + _airbyte_children_hashid, + owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_owner_hashid +from "test_normalization"._airbyte_test_normalization."unnest_alias_children_owner_ab3" +-- owner at unnest_alias/children/owner from "test_normalization".test_normalization."unnest_alias_children" +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."unnest_alias_children_owner__dbt_tmp" FROM + "test_normalization".test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_unnest_alias_children_owner__dbt_tmp_cci' + AND object_id=object_id('test_normalization_unnest_alias_children_owner__dbt_tmp') + ) + DROP index test_normalization.unnest_alias_children_owner__dbt_tmp.test_normalization_unnest_alias_children_owner__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_unnest_alias_children_owner__dbt_tmp_cci + ON test_normalization.unnest_alias_children_owner__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql new file mode 100644 index 000000000000..b6091947b13a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp"','U') is not null + begin + drop table test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view" as + +-- Final base SQL model +select + id, + "date", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_simple_strea__nto_long_names_hashid +from "test_normalization"._airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab3" +-- simple_stream_with_na__lting_into_long_names from "test_normalization".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp" FROM + "test_normalization".test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_namespace_simple_stream_with_na__lting_into_long_names__dbt_tmp_cci' + AND object_id=object_id('test_normalization_namespace_simple_stream_with_na__lting_into_long_names__dbt_tmp') + ) + DROP index test_normalization_namespace.simple_stream_with_na__lting_into_long_names__dbt_tmp.test_normalization_namespace_simple_stream_with_na__lting_into_long_names__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_namespace_simple_stream_with_na__lting_into_long_names__dbt_tmp_cci + ON test_normalization_namespace.simple_stream_with_na__lting_into_long_names__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql new file mode 100644 index 000000000000..871fcf869df0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql @@ -0,0 +1,21 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as + VARCHAR(max)), ''''), ''-'', coalesce(cast("partition" as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_nested_strea__nto_long_names_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab2" tmp +-- nested_stream_with_co__lting_into_long_names +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql deleted file mode 100644 index 2a936635d1f1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."dedup_cdc_excluded_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_value(_airbyte_data, ''$."name"'') as name, - json_value(_airbyte_data, ''$."column`_''''with\"_quotes"'') as "column`_''with""_quotes", - json_value(_airbyte_data, ''$."_ab_cdc_lsn"'') as _ab_cdc_lsn, - json_value(_airbyte_data, ''$."_ab_cdc_updated_at"'') as _ab_cdc_updated_at, - json_value(_airbyte_data, ''$."_ab_cdc_deleted_at"'') as _ab_cdc_deleted_at, - _airbyte_emitted_at -from "test_normalization".test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias --- dedup_cdc_excluded - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql deleted file mode 100644 index a289f83abc82..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,26 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."dedup_cdc_excluded_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - cast(name as - VARCHAR(max)) as name, - cast("column`_''with""_quotes" as - VARCHAR(max)) as "column`_''with""_quotes", - cast(_ab_cdc_lsn as - float -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float -) as _ab_cdc_deleted_at, - _airbyte_emitted_at -from "test_normalization"._airbyte_test_normalization."dedup_cdc_excluded_ab1" --- dedup_cdc_excluded - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 3dc43da1b4b0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,22 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."dedup_cdc_excluded_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''), ''-'', coalesce(cast(name as - VARCHAR(max)), ''''), ''-'', coalesce(cast("column`_''with""_quotes" as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_lsn as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_updated_at as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_deleted_at as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."dedup_cdc_excluded_ab2" tmp --- dedup_cdc_excluded - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql deleted file mode 100644 index 241474634fb2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql +++ /dev/null @@ -1,14 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."dedup_cdc_excluded_ab4__dbt_tmp" as - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_cdc_excluded_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from "test_normalization"._airbyte_test_normalization."dedup_cdc_excluded_ab3" tmp --- dedup_cdc_excluded from "test_normalization".test_normalization._airbyte_raw_dedup_cdc_excluded - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql deleted file mode 100644 index 84e8fa8a4767..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."dedup_exchange_rate_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_value(_airbyte_data, ''$."currency"'') as currency, - json_value(_airbyte_data, ''$."date"'') as "date", - json_value(_airbyte_data, ''$."timestamp_col"'') as timestamp_col, - json_value(_airbyte_data, ''$."HKD@spéçiäl & characters"'') as "HKD@spéçiäl & characters", - json_value(_airbyte_data, ''$."HKD_special___characters"'') as hkd_special___characters, - json_value(_airbyte_data, ''$."NZD"'') as nzd, - json_value(_airbyte_data, ''$."USD"'') as usd, - _airbyte_emitted_at -from "test_normalization".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias --- dedup_exchange_rate - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql deleted file mode 100644 index fc83ce6c39bc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ /dev/null @@ -1,28 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."dedup_exchange_rate_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - cast(currency as - VARCHAR(max)) as currency, - try_parse(nullif("date", '''') as date) as "date", - try_parse(nullif(timestamp_col, '''') as datetime) as timestamp_col, - cast("HKD@spéçiäl & characters" as - float -) as "HKD@spéçiäl & characters", - cast(hkd_special___characters as - VARCHAR(max)) as hkd_special___characters, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - _airbyte_emitted_at -from "test_normalization"._airbyte_test_normalization."dedup_exchange_rate_ab1" --- dedup_exchange_rate - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql deleted file mode 100644 index 3411a2a70ae4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql +++ /dev/null @@ -1,24 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."dedup_exchange_rate_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as - VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as - VARCHAR(max)), ''''), ''-'', coalesce(cast(timestamp_col as - VARCHAR(max)), ''''), ''-'', coalesce(cast("HKD@spéçiäl & characters" as - VARCHAR(max)), ''''), ''-'', coalesce(cast(hkd_special___characters as - VARCHAR(max)), ''''), ''-'', coalesce(cast(nzd as - VARCHAR(max)), ''''), ''-'', coalesce(cast(usd as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_dedup_exchange_rate_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."dedup_exchange_rate_ab2" tmp --- dedup_exchange_rate - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql deleted file mode 100644 index 086bfc9a91d7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql +++ /dev/null @@ -1,14 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."dedup_exchange_rate_ab4__dbt_tmp" as - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_exchange_rate_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from "test_normalization"._airbyte_test_normalization."dedup_exchange_rate_ab3" tmp --- dedup_exchange_rate from "test_normalization".test_normalization._airbyte_raw_dedup_exchange_rate - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab1.sql deleted file mode 100644 index 94dc9cd399d0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."exchange_rate_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_value(_airbyte_data, ''$."currency"'') as currency, - json_value(_airbyte_data, ''$."date"'') as "date", - json_value(_airbyte_data, ''$."timestamp_col"'') as timestamp_col, - json_value(_airbyte_data, ''$."HKD@spéçiäl & characters"'') as "HKD@spéçiäl & characters", - json_value(_airbyte_data, ''$."HKD_special___characters"'') as hkd_special___characters, - json_value(_airbyte_data, ''$."NZD"'') as nzd, - json_value(_airbyte_data, ''$."USD"'') as usd, - _airbyte_emitted_at -from "test_normalization".test_normalization._airbyte_raw_exchange_rate as table_alias --- exchange_rate - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab2.sql deleted file mode 100644 index f24614e14702..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ /dev/null @@ -1,28 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."exchange_rate_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - cast(currency as - VARCHAR(max)) as currency, - try_parse(nullif("date", '''') as date) as "date", - try_parse(nullif(timestamp_col, '''') as datetime) as timestamp_col, - cast("HKD@spéçiäl & characters" as - float -) as "HKD@spéçiäl & characters", - cast(hkd_special___characters as - VARCHAR(max)) as hkd_special___characters, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - _airbyte_emitted_at -from "test_normalization"._airbyte_test_normalization."exchange_rate_ab1" --- exchange_rate - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab3.sql deleted file mode 100644 index bf8f496f0b61..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ /dev/null @@ -1,24 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."exchange_rate_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as - VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as - VARCHAR(max)), ''''), ''-'', coalesce(cast(timestamp_col as - VARCHAR(max)), ''''), ''-'', coalesce(cast("HKD@spéçiäl & characters" as - VARCHAR(max)), ''''), ''-'', coalesce(cast(hkd_special___characters as - VARCHAR(max)), ''''), ''-'', coalesce(cast(nzd as - VARCHAR(max)), ''''), ''-'', coalesce(cast(usd as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_exchange_rate_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."exchange_rate_ab2" tmp --- exchange_rate - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql deleted file mode 100644 index 95187f4652a1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."pos_dedup_cdcx_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_value(_airbyte_data, ''$."name"'') as name, - json_value(_airbyte_data, ''$."_ab_cdc_lsn"'') as _ab_cdc_lsn, - json_value(_airbyte_data, ''$."_ab_cdc_updated_at"'') as _ab_cdc_updated_at, - json_value(_airbyte_data, ''$."_ab_cdc_deleted_at"'') as _ab_cdc_deleted_at, - json_value(_airbyte_data, ''$."_ab_cdc_log_pos"'') as _ab_cdc_log_pos, - _airbyte_emitted_at -from "test_normalization".test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias --- pos_dedup_cdcx - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql deleted file mode 100644 index b8d7c54651e2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ /dev/null @@ -1,27 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."pos_dedup_cdcx_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - cast(name as - VARCHAR(max)) as name, - cast(_ab_cdc_lsn as - float -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float -) as _ab_cdc_deleted_at, - cast(_ab_cdc_log_pos as - float -) as _ab_cdc_log_pos, - _airbyte_emitted_at -from "test_normalization"._airbyte_test_normalization."pos_dedup_cdcx_ab1" --- pos_dedup_cdcx - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index 13903db7e117..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,22 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."pos_dedup_cdcx_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''), ''-'', coalesce(cast(name as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_lsn as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_updated_at as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_deleted_at as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_log_pos as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_pos_dedup_cdcx_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."pos_dedup_cdcx_ab2" tmp --- pos_dedup_cdcx - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql deleted file mode 100644 index 0ff1550ed499..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql +++ /dev/null @@ -1,14 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."pos_dedup_cdcx_ab4__dbt_tmp" as - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_pos_dedup_cdcx_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from "test_normalization"._airbyte_test_normalization."pos_dedup_cdcx_ab3" tmp --- pos_dedup_cdcx from "test_normalization".test_normalization._airbyte_raw_pos_dedup_cdcx - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index 7d3a90565705..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,60 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."dedup_cdc_excluded__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."dedup_cdc_excluded__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."dedup_cdc_excluded__dbt_tmp"','U') is not null - begin - drop table test_normalization."dedup_cdc_excluded__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."dedup_cdc_excluded__dbt_tmp_temp_view" as - --- Final base SQL model -select - id, - name, - "column`_''with""_quotes", - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from "test_normalization".test_normalization."dedup_cdc_excluded_scd" --- dedup_cdc_excluded from "test_normalization".test_normalization._airbyte_raw_dedup_cdc_excluded -where _airbyte_active_row = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."dedup_cdc_excluded__dbt_tmp" FROM - "test_normalization".test_normalization."dedup_cdc_excluded__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."dedup_cdc_excluded__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."dedup_cdc_excluded__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_dedup_cdc_excluded__dbt_tmp_cci' - AND object_id=object_id('test_normalization_dedup_cdc_excluded__dbt_tmp') - ) - DROP index test_normalization.dedup_cdc_excluded__dbt_tmp.test_normalization_dedup_cdc_excluded__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_dedup_cdc_excluded__dbt_tmp_cci - ON test_normalization.dedup_cdc_excluded__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index 1b50e2ded65d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,69 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."dedup_cdc_excluded_scd__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."dedup_cdc_excluded_scd__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."dedup_cdc_excluded_scd__dbt_tmp"','U') is not null - begin - drop table test_normalization."dedup_cdc_excluded_scd__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."dedup_cdc_excluded_scd__dbt_tmp_temp_view" as - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - "column`_''with""_quotes", - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at desc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at desc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from "test_normalization"._airbyte_test_normalization."dedup_cdc_excluded_ab4" --- dedup_cdc_excluded from "test_normalization".test_normalization._airbyte_raw_dedup_cdc_excluded -where _airbyte_row_num = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."dedup_cdc_excluded_scd__dbt_tmp" FROM - "test_normalization".test_normalization."dedup_cdc_excluded_scd__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."dedup_cdc_excluded_scd__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."dedup_cdc_excluded_scd__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_dedup_cdc_excluded_scd__dbt_tmp_cci' - AND object_id=object_id('test_normalization_dedup_cdc_excluded_scd__dbt_tmp') - ) - DROP index test_normalization.dedup_cdc_excluded_scd__dbt_tmp.test_normalization_dedup_cdc_excluded_scd__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_dedup_cdc_excluded_scd__dbt_tmp_cci - ON test_normalization.dedup_cdc_excluded_scd__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql deleted file mode 100644 index 8d67ed36355d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql +++ /dev/null @@ -1,62 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."dedup_exchange_rate__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."dedup_exchange_rate__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."dedup_exchange_rate__dbt_tmp"','U') is not null - begin - drop table test_normalization."dedup_exchange_rate__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."dedup_exchange_rate__dbt_tmp_temp_view" as - --- Final base SQL model -select - id, - currency, - "date", - timestamp_col, - "HKD@spéçiäl & characters", - hkd_special___characters, - nzd, - usd, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from "test_normalization".test_normalization."dedup_exchange_rate_scd" --- dedup_exchange_rate from "test_normalization".test_normalization._airbyte_raw_dedup_exchange_rate -where _airbyte_active_row = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."dedup_exchange_rate__dbt_tmp" FROM - "test_normalization".test_normalization."dedup_exchange_rate__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."dedup_exchange_rate__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."dedup_exchange_rate__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_dedup_exchange_rate__dbt_tmp_cci' - AND object_id=object_id('test_normalization_dedup_exchange_rate__dbt_tmp') - ) - DROP index test_normalization.dedup_exchange_rate__dbt_tmp.test_normalization_dedup_exchange_rate__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_dedup_exchange_rate__dbt_tmp_cci - ON test_normalization.dedup_exchange_rate__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql deleted file mode 100644 index c6f7f77a55a7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql +++ /dev/null @@ -1,73 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."dedup_exchange_rate_scd__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."dedup_exchange_rate_scd__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."dedup_exchange_rate_scd__dbt_tmp"','U') is not null - begin - drop table test_normalization."dedup_exchange_rate_scd__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."dedup_exchange_rate_scd__dbt_tmp_temp_view" as - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - currency, - "date", - timestamp_col, - "HKD@spéçiäl & characters", - hkd_special___characters, - nzd, - usd, - "date" as _airbyte_start_at, - lag("date") over ( - partition by id, currency, cast(nzd as - VARCHAR(max)) - order by "date" desc, "date" desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag("date") over ( - partition by id, currency, cast(nzd as - VARCHAR(max)) - order by "date" desc, "date" desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from "test_normalization"._airbyte_test_normalization."dedup_exchange_rate_ab4" --- dedup_exchange_rate from "test_normalization".test_normalization._airbyte_raw_dedup_exchange_rate -where _airbyte_row_num = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."dedup_exchange_rate_scd__dbt_tmp" FROM - "test_normalization".test_normalization."dedup_exchange_rate_scd__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."dedup_exchange_rate_scd__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."dedup_exchange_rate_scd__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_dedup_exchange_rate_scd__dbt_tmp_cci' - AND object_id=object_id('test_normalization_dedup_exchange_rate_scd__dbt_tmp') - ) - DROP index test_normalization.dedup_exchange_rate_scd__dbt_tmp.test_normalization_dedup_exchange_rate_scd__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_dedup_exchange_rate_scd__dbt_tmp_cci - ON test_normalization.dedup_exchange_rate_scd__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql deleted file mode 100644 index 4c6508f2840c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,61 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."exchange_rate__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."exchange_rate__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."exchange_rate__dbt_tmp"','U') is not null - begin - drop table test_normalization."exchange_rate__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."exchange_rate__dbt_tmp_temp_view" as - --- Final base SQL model -select - id, - currency, - "date", - timestamp_col, - "HKD@spéçiäl & characters", - hkd_special___characters, - nzd, - usd, - _airbyte_emitted_at, - _airbyte_exchange_rate_hashid -from "test_normalization"._airbyte_test_normalization."exchange_rate_ab3" --- exchange_rate from "test_normalization".test_normalization._airbyte_raw_exchange_rate - '); - - SELECT * INTO "test_normalization".test_normalization."exchange_rate__dbt_tmp" FROM - "test_normalization".test_normalization."exchange_rate__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."exchange_rate__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."exchange_rate__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_exchange_rate__dbt_tmp_cci' - AND object_id=object_id('test_normalization_exchange_rate__dbt_tmp') - ) - DROP index test_normalization.exchange_rate__dbt_tmp.test_normalization_exchange_rate__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_exchange_rate__dbt_tmp_cci - ON test_normalization.exchange_rate__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql deleted file mode 100644 index 0db7ca237be8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ /dev/null @@ -1,60 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."pos_dedup_cdcx__dbt_tmp"','U') is not null - begin - drop table test_normalization."pos_dedup_cdcx__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view" as - --- Final base SQL model -select - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid -from "test_normalization".test_normalization."pos_dedup_cdcx_scd" --- pos_dedup_cdcx from "test_normalization".test_normalization._airbyte_raw_pos_dedup_cdcx -where _airbyte_active_row = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."pos_dedup_cdcx__dbt_tmp" FROM - "test_normalization".test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_pos_dedup_cdcx__dbt_tmp_cci' - AND object_id=object_id('test_normalization_pos_dedup_cdcx__dbt_tmp') - ) - DROP index test_normalization.pos_dedup_cdcx__dbt_tmp.test_normalization_pos_dedup_cdcx__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_pos_dedup_cdcx__dbt_tmp_cci - ON test_normalization.pos_dedup_cdcx__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index e66831879ebe..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,69 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."pos_dedup_cdcx_scd__dbt_tmp"','U') is not null - begin - drop table test_normalization."pos_dedup_cdcx_scd__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" as - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at desc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at desc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid -from "test_normalization"._airbyte_test_normalization."pos_dedup_cdcx_ab4" --- pos_dedup_cdcx from "test_normalization".test_normalization._airbyte_raw_pos_dedup_cdcx -where _airbyte_row_num = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."pos_dedup_cdcx_scd__dbt_tmp" FROM - "test_normalization".test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_pos_dedup_cdcx_scd__dbt_tmp_cci' - AND object_id=object_id('test_normalization_pos_dedup_cdcx_scd__dbt_tmp') - ) - DROP index test_normalization.pos_dedup_cdcx_scd__dbt_tmp.test_normalization_pos_dedup_cdcx_scd__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_pos_dedup_cdcx_scd__dbt_tmp_cci - ON test_normalization.pos_dedup_cdcx_scd__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..7ca4f64da70a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."dedup_cdc_excluded_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."name"'') as name, + json_value(_airbyte_data, ''$."_ab_cdc_lsn"'') as _ab_cdc_lsn, + json_value(_airbyte_data, ''$."_ab_cdc_updated_at"'') as _ab_cdc_updated_at, + json_value(_airbyte_data, ''$."_ab_cdc_deleted_at"'') as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias +-- dedup_cdc_excluded +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..071d2319c025 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -0,0 +1,28 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."dedup_cdc_excluded_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(name as + VARCHAR(max)) as name, + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."dedup_cdc_excluded_ab1" +-- dedup_cdc_excluded +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql new file mode 100644 index 000000000000..62877d8eede6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -0,0 +1,22 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."dedup_exchange_rate_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."currency"'') as currency, + json_value(_airbyte_data, ''$."date"'') as "date", + json_value(_airbyte_data, ''$."timestamp_col"'') as timestamp_col, + json_value(_airbyte_data, ''$."HKD@spéçiäl & characters"'') as "HKD@spéçiäl & characters", + json_value(_airbyte_data, ''$."HKD_special___characters"'') as hkd_special___characters, + json_value(_airbyte_data, ''$."NZD"'') as nzd, + json_value(_airbyte_data, ''$."USD"'') as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql new file mode 100644 index 000000000000..b871d6e41b0b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -0,0 +1,32 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."dedup_exchange_rate_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(currency as + VARCHAR(max)) as currency, + try_parse(nullif("date", '''') as date) as "date", + try_parse(nullif(timestamp_col, '''') as datetime) as timestamp_col, + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(hkd_special___characters as + VARCHAR(max)) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."dedup_exchange_rate_ab1" +-- dedup_exchange_rate +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql new file mode 100644 index 000000000000..31eb5bb200ab --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -0,0 +1,22 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."exchange_rate_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."currency"'') as currency, + json_value(_airbyte_data, ''$."date"'') as "date", + json_value(_airbyte_data, ''$."timestamp_col"'') as timestamp_col, + json_value(_airbyte_data, ''$."HKD@spéçiäl & characters"'') as "HKD@spéçiäl & characters", + json_value(_airbyte_data, ''$."HKD_special___characters"'') as hkd_special___characters, + json_value(_airbyte_data, ''$."NZD"'') as nzd, + json_value(_airbyte_data, ''$."USD"'') as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql new file mode 100644 index 000000000000..bf6e40566af3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -0,0 +1,32 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."exchange_rate_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(currency as + VARCHAR(max)) as currency, + try_parse(nullif("date", '''') as date) as "date", + try_parse(nullif(timestamp_col, '''') as datetime) as timestamp_col, + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(hkd_special___characters as + VARCHAR(max)) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."exchange_rate_ab1" +-- exchange_rate +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql new file mode 100644 index 000000000000..13c53bf0f858 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -0,0 +1,26 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."exchange_rate_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as + VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as + VARCHAR(max)), ''''), ''-'', coalesce(cast(timestamp_col as + VARCHAR(max)), ''''), ''-'', coalesce(cast("HKD@spéçiäl & characters" as + VARCHAR(max)), ''''), ''-'', coalesce(cast(hkd_special___characters as + VARCHAR(max)), ''''), ''-'', coalesce(cast(nzd as + VARCHAR(max)), ''''), ''-'', coalesce(cast(usd as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_exchange_rate_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."exchange_rate_ab2" tmp +-- exchange_rate +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql new file mode 100644 index 000000000000..d210ebebcdfc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."pos_dedup_cdcx_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."name"'') as name, + json_value(_airbyte_data, ''$."_ab_cdc_lsn"'') as _ab_cdc_lsn, + json_value(_airbyte_data, ''$."_ab_cdc_updated_at"'') as _ab_cdc_updated_at, + json_value(_airbyte_data, ''$."_ab_cdc_deleted_at"'') as _ab_cdc_deleted_at, + json_value(_airbyte_data, ''$."_ab_cdc_log_pos"'') as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias +-- pos_dedup_cdcx +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql new file mode 100644 index 000000000000..d23785697728 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -0,0 +1,30 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."pos_dedup_cdcx_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(name as + VARCHAR(max)) as name, + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + cast(_ab_cdc_log_pos as + float +) as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."pos_dedup_cdcx_ab1" +-- pos_dedup_cdcx +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..7c05f028e95e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,15 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..362c8ff7d323 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,17 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1" +-- renamed_dedup_cdc_excluded +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..b98aeb069663 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,133 @@ + + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_cdc_excluded_scd_temp_view"','V') is not null + begin + drop view test_normalization."dedup_cdc_excluded_scd_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_cdc_excluded_scd"','U') is not null + begin + drop table test_normalization."dedup_cdc_excluded_scd" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."dedup_cdc_excluded_scd_temp_view" as + +with + +input_data as ( + select * + from "test_normalization"._airbyte_test_normalization."dedup_cdc_excluded_ab3" + -- dedup_cdc_excluded from "test_normalization".test_normalization._airbyte_raw_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as + VARCHAR(max)), cast(_ab_cdc_updated_at as + VARCHAR(max)) + order by _airbyte_ab_id + ) as _airbyte_row_num, + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_unique_key as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_start_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_emitted_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_deleted_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_updated_at as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."dedup_cdc_excluded_scd" FROM + "test_normalization".test_normalization."dedup_cdc_excluded_scd_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_cdc_excluded_scd_temp_view"','V') is not null + begin + drop view test_normalization."dedup_cdc_excluded_scd_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_dedup_cdc_excluded_scd_cci' + AND object_id=object_id('test_normalization_dedup_cdc_excluded_scd') + ) + DROP index test_normalization.dedup_cdc_excluded_scd.test_normalization_dedup_cdc_excluded_scd_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_dedup_cdc_excluded_scd_cci + ON test_normalization.dedup_cdc_excluded_scd + + + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..60c4f8d8eab2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,139 @@ + + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_exchange_rate_scd_temp_view"','V') is not null + begin + drop view test_normalization."dedup_exchange_rate_scd_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_exchange_rate_scd"','U') is not null + begin + drop table test_normalization."dedup_exchange_rate_scd" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."dedup_exchange_rate_scd_temp_view" as + +with + +input_data as ( + select * + from "test_normalization"._airbyte_test_normalization."dedup_exchange_rate_ab3" + -- dedup_exchange_rate from "test_normalization".test_normalization._airbyte_raw_dedup_exchange_rate +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as + VARCHAR(max)), ''''), ''-'', coalesce(cast(nzd as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_unique_key, + id, + currency, + "date", + timestamp_col, + "HKD@spéçiäl & characters", + hkd_special___characters, + nzd, + usd, + "date" as _airbyte_start_at, + lag("date") over ( + partition by id, currency, cast(nzd as + VARCHAR(max)) + order by + "date" desc, + "date" desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag("date") over ( + partition by id, currency, cast(nzd as + VARCHAR(max)) + order by + "date" desc, + "date" desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_exchange_rate_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_unique_key as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_start_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_emitted_at as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + currency, + "date", + timestamp_col, + "HKD@spéçiäl & characters", + hkd_special___characters, + nzd, + usd, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from dedup_data where _airbyte_row_num = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."dedup_exchange_rate_scd" FROM + "test_normalization".test_normalization."dedup_exchange_rate_scd_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_exchange_rate_scd_temp_view"','V') is not null + begin + drop view test_normalization."dedup_exchange_rate_scd_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_dedup_exchange_rate_scd_cci' + AND object_id=object_id('test_normalization_dedup_exchange_rate_scd') + ) + DROP index test_normalization.dedup_exchange_rate_scd.test_normalization_dedup_exchange_rate_scd_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_dedup_exchange_rate_scd_cci + ON test_normalization.dedup_exchange_rate_scd + + + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..967080a2b6ae --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,121 @@ + + + + USE [test_normalization]; + if object_id ('test_normalization."renamed_dedup_cdc_excluded_scd_temp_view"','V') is not null + begin + drop view test_normalization."renamed_dedup_cdc_excluded_scd_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."renamed_dedup_cdc_excluded_scd"','U') is not null + begin + drop table test_normalization."renamed_dedup_cdc_excluded_scd" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."renamed_dedup_cdc_excluded_scd_temp_view" as + +with + +input_data as ( + select * + from "test_normalization"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3" + -- renamed_dedup_cdc_excluded from "test_normalization".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_unique_key, + id, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_renamed_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_unique_key as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_start_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_emitted_at as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."renamed_dedup_cdc_excluded_scd" FROM + "test_normalization".test_normalization."renamed_dedup_cdc_excluded_scd_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."renamed_dedup_cdc_excluded_scd_temp_view"','V') is not null + begin + drop view test_normalization."renamed_dedup_cdc_excluded_scd_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_renamed_dedup_cdc_excluded_scd_cci' + AND object_id=object_id('test_normalization_renamed_dedup_cdc_excluded_scd') + ) + DROP index test_normalization.renamed_dedup_cdc_excluded_scd.test_normalization_renamed_dedup_cdc_excluded_scd_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_renamed_dedup_cdc_excluded_scd_cci + ON test_normalization.renamed_dedup_cdc_excluded_scd + + + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql new file mode 100644 index 000000000000..c2c86219d046 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -0,0 +1,67 @@ + + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_cdc_excluded_temp_view"','V') is not null + begin + drop view test_normalization."dedup_cdc_excluded_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_cdc_excluded"','U') is not null + begin + drop table test_normalization."dedup_cdc_excluded" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."dedup_cdc_excluded_temp_view" as + +-- Final base SQL model +select + _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from "test_normalization".test_normalization."dedup_cdc_excluded_scd" +-- dedup_cdc_excluded from "test_normalization".test_normalization._airbyte_raw_dedup_cdc_excluded +where 1 = 1 +and _airbyte_active_row = 1 + + '); + + SELECT * INTO "test_normalization".test_normalization."dedup_cdc_excluded" FROM + "test_normalization".test_normalization."dedup_cdc_excluded_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_cdc_excluded_temp_view"','V') is not null + begin + drop view test_normalization."dedup_cdc_excluded_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_dedup_cdc_excluded_cci' + AND object_id=object_id('test_normalization_dedup_cdc_excluded') + ) + DROP index test_normalization.dedup_cdc_excluded.test_normalization_dedup_cdc_excluded_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_dedup_cdc_excluded_cci + ON test_normalization.dedup_cdc_excluded + + + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..e982dbeed83c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,70 @@ + + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_exchange_rate_temp_view"','V') is not null + begin + drop view test_normalization."dedup_exchange_rate_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_exchange_rate"','U') is not null + begin + drop table test_normalization."dedup_exchange_rate" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."dedup_exchange_rate_temp_view" as + +-- Final base SQL model +select + _airbyte_unique_key, + id, + currency, + "date", + timestamp_col, + "HKD@spéçiäl & characters", + hkd_special___characters, + nzd, + usd, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from "test_normalization".test_normalization."dedup_exchange_rate_scd" +-- dedup_exchange_rate from "test_normalization".test_normalization._airbyte_raw_dedup_exchange_rate +where 1 = 1 +and _airbyte_active_row = 1 + + '); + + SELECT * INTO "test_normalization".test_normalization."dedup_exchange_rate" FROM + "test_normalization".test_normalization."dedup_exchange_rate_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_exchange_rate_temp_view"','V') is not null + begin + drop view test_normalization."dedup_exchange_rate_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_dedup_exchange_rate_cci' + AND object_id=object_id('test_normalization_dedup_exchange_rate') + ) + DROP index test_normalization.dedup_exchange_rate.test_normalization_dedup_exchange_rate_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_dedup_exchange_rate_cci + ON test_normalization.dedup_exchange_rate + + + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..55dca5724089 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -0,0 +1,68 @@ + + + + USE [test_normalization]; + if object_id ('test_normalization."exchange_rate_temp_view"','V') is not null + begin + drop view test_normalization."exchange_rate_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."exchange_rate"','U') is not null + begin + drop table test_normalization."exchange_rate" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."exchange_rate_temp_view" as + +-- Final base SQL model +select + id, + currency, + "date", + timestamp_col, + "HKD@spéçiäl & characters", + hkd_special___characters, + nzd, + usd, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from "test_normalization"._airbyte_test_normalization."exchange_rate_ab3" +-- exchange_rate from "test_normalization".test_normalization._airbyte_raw_exchange_rate +where 1 = 1 + + '); + + SELECT * INTO "test_normalization".test_normalization."exchange_rate" FROM + "test_normalization".test_normalization."exchange_rate_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."exchange_rate_temp_view"','V') is not null + begin + drop view test_normalization."exchange_rate_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_exchange_rate_cci' + AND object_id=object_id('test_normalization_exchange_rate') + ) + DROP index test_normalization.exchange_rate.test_normalization_exchange_rate_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_exchange_rate_cci + ON test_normalization.exchange_rate + + + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..8ddf455c5ec9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,63 @@ + + + + USE [test_normalization]; + if object_id ('test_normalization."renamed_dedup_cdc_excluded_temp_view"','V') is not null + begin + drop view test_normalization."renamed_dedup_cdc_excluded_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."renamed_dedup_cdc_excluded"','U') is not null + begin + drop table test_normalization."renamed_dedup_cdc_excluded" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."renamed_dedup_cdc_excluded_temp_view" as + +-- Final base SQL model +select + _airbyte_unique_key, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from "test_normalization".test_normalization."renamed_dedup_cdc_excluded_scd" +-- renamed_dedup_cdc_excluded from "test_normalization".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded +where 1 = 1 +and _airbyte_active_row = 1 + + '); + + SELECT * INTO "test_normalization".test_normalization."renamed_dedup_cdc_excluded" FROM + "test_normalization".test_normalization."renamed_dedup_cdc_excluded_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."renamed_dedup_cdc_excluded_temp_view"','V') is not null + begin + drop view test_normalization."renamed_dedup_cdc_excluded_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_renamed_dedup_cdc_excluded_cci' + AND object_id=object_id('test_normalization_renamed_dedup_cdc_excluded') + ) + DROP index test_normalization.renamed_dedup_cdc_excluded.test_normalization_renamed_dedup_cdc_excluded_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_renamed_dedup_cdc_excluded_cci + ON test_normalization.renamed_dedup_cdc_excluded + + + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..749435773423 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,134 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."pos_dedup_cdcx_scd__dbt_tmp"','U') is not null + begin + drop table test_normalization."pos_dedup_cdcx_scd__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" as + +with + +input_data as ( + select * + from "test_normalization"._airbyte_test_normalization."pos_dedup_cdcx_ab3" + -- pos_dedup_cdcx from "test_normalization".test_normalization._airbyte_raw_pos_dedup_cdcx +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as + VARCHAR(max)), cast(_ab_cdc_updated_at as + VARCHAR(max)), cast(_ab_cdc_log_pos as + VARCHAR(max)) + order by _airbyte_ab_id + ) as _airbyte_row_num, + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_unique_key as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_start_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_emitted_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_deleted_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_updated_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_log_pos as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."pos_dedup_cdcx_scd__dbt_tmp" FROM + "test_normalization".test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_pos_dedup_cdcx_scd__dbt_tmp_cci' + AND object_id=object_id('test_normalization_pos_dedup_cdcx_scd__dbt_tmp') + ) + DROP index test_normalization.pos_dedup_cdcx_scd__dbt_tmp.test_normalization_pos_dedup_cdcx_scd__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_pos_dedup_cdcx_scd__dbt_tmp_cci + ON test_normalization.pos_dedup_cdcx_scd__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql new file mode 100644 index 000000000000..846de1108761 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -0,0 +1,64 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."pos_dedup_cdcx__dbt_tmp"','U') is not null + begin + drop table test_normalization."pos_dedup_cdcx__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view" as + +-- Final base SQL model +select + _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from "test_normalization".test_normalization."pos_dedup_cdcx_scd" +-- pos_dedup_cdcx from "test_normalization".test_normalization._airbyte_raw_pos_dedup_cdcx +where 1 = 1 +and _airbyte_active_row = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."pos_dedup_cdcx__dbt_tmp" FROM + "test_normalization".test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_pos_dedup_cdcx__dbt_tmp_cci' + AND object_id=object_id('test_normalization_pos_dedup_cdcx__dbt_tmp') + ) + DROP index test_normalization.pos_dedup_cdcx__dbt_tmp.test_normalization_pos_dedup_cdcx__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_pos_dedup_cdcx__dbt_tmp_cci + ON test_normalization.pos_dedup_cdcx__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..71b4b90ce304 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,23 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."dedup_cdc_excluded_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(name as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_lsn as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_updated_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_deleted_at as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_dedup_cdc_excluded_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."dedup_cdc_excluded_ab2" tmp +-- dedup_cdc_excluded +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..021c2a7b88f4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,26 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."dedup_exchange_rate_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as + VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as + VARCHAR(max)), ''''), ''-'', coalesce(cast(timestamp_col as + VARCHAR(max)), ''''), ''-'', coalesce(cast("HKD@spéçiäl & characters" as + VARCHAR(max)), ''''), ''-'', coalesce(cast(hkd_special___characters as + VARCHAR(max)), ''''), ''-'', coalesce(cast(nzd as + VARCHAR(max)), ''''), ''-'', coalesce(cast(usd as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."dedup_exchange_rate_ab2" tmp +-- dedup_exchange_rate +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql new file mode 100644 index 000000000000..fe243c2ddc74 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -0,0 +1,23 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."pos_dedup_cdcx_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(name as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_lsn as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_updated_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_deleted_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_log_pos as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_pos_dedup_cdcx_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."pos_dedup_cdcx_ab2" tmp +-- pos_dedup_cdcx +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..6781d88c4b0f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2" tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql deleted file mode 100644 index 6144a13617e8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql +++ /dev/null @@ -1,43 +0,0 @@ -with table_row_counts as ( - select distinct count(*) as row_count, 10 as expected_count - from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} -union all - select distinct count(*) as row_count, 10 as expected_count - from {{ ref('exchange_rate') }} - -union all - - select distinct count(*) as row_count, 10 as expected_count - from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -union all - select distinct count(*) as row_count, 10 as expected_count - from {{ ref('dedup_exchange_rate_scd') }} -union all - select distinct count(*) as row_count, 5 as expected_count - from {{ ref('dedup_exchange_rate') }} - -union all - - select distinct count(*) as row_count, 8 as expected_count - from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -union all - select distinct count(*) as row_count, 8 as expected_count - from {{ ref('dedup_cdc_excluded_scd') }} -union all - select distinct count(*) as row_count, 4 as expected_count - from {{ ref('dedup_cdc_excluded') }} - -union all - - select distinct count(*) as row_count, 8 as expected_count - from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -union all - select distinct count(*) as row_count, 8 as expected_count - from {{ ref('pos_dedup_cdcx_scd') }} -union all - select distinct count(*) as row_count, 3 as expected_count - from {{ ref('pos_dedup_cdcx') }} -) -select * -from table_row_counts -where row_count != expected_count diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql index 71f6448f6532..bc0150c8df4e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -1,13 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['name'], ['name']) }} as name, - {{ json_extract_scalar('_airbyte_data', ['column`_\'with"_quotes'], ['column___with__quotes']) }} as {{ adapter.quote('column`_\'with""_quotes') }}, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_lsn'], ['_ab_cdc_lsn']) }} as _ab_cdc_lsn, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} as table_alias -- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql index 2f1c85715033..37994c4823f9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -1,13 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, cast(name as {{ dbt_utils.type_string() }}) as name, - cast({{ adapter.quote('column`_\'with""_quotes') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('column`_\'with""_quotes') }}, cast(_ab_cdc_lsn as {{ dbt_utils.type_float() }}) as _ab_cdc_lsn, cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('dedup_cdc_excluded_ab1') }} -- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 4e6d97a65757..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'name', - adapter.quote('column`_\'with""_quotes'), - '_ab_cdc_lsn', - '_ab_cdc_updated_at', - '_ab_cdc_deleted_at', - ]) }} as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from {{ ref('dedup_cdc_excluded_ab2') }} tmp --- dedup_cdc_excluded - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql deleted file mode 100644 index aa34179c3c5c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_cdc_excluded_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('dedup_cdc_excluded_ab3') }} tmp --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql index c9a6865ef7fd..5ed2b660ee09 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, @@ -9,7 +13,11 @@ select {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as hkd_special___characters, {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as nzd, {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} as table_alias -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index 539db2675e74..c20798c70b01 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -9,7 +13,11 @@ select cast(hkd_special___characters as {{ dbt_utils.type_string() }}) as hkd_special___characters, cast(nzd as {{ dbt_utils.type_float() }}) as nzd, cast(usd as {{ dbt_utils.type_float() }}) as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('dedup_exchange_rate_ab1') }} -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql deleted file mode 100644 index 3e68860804dc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'currency', - adapter.quote('date'), - 'timestamp_col', - adapter.quote('HKD@spéçiäl & characters'), - 'hkd_special___characters', - 'nzd', - 'usd', - ]) }} as _airbyte_dedup_exchange_rate_hashid, - tmp.* -from {{ ref('dedup_exchange_rate_ab2') }} tmp --- dedup_exchange_rate - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql deleted file mode 100644 index ef5f6e52d60e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_exchange_rate_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('dedup_exchange_rate_ab3') }} tmp --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql index 3e66f6d0b4fc..c20f382cfbc0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, @@ -9,7 +13,11 @@ select {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as hkd_special___characters, {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as nzd, {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} as table_alias -- exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql index 22521fcb655a..b737134cc6eb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -9,7 +13,11 @@ select cast(hkd_special___characters as {{ dbt_utils.type_string() }}) as hkd_special___characters, cast(nzd as {{ dbt_utils.type_float() }}) as nzd, cast(usd as {{ dbt_utils.type_float() }}) as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('exchange_rate_ab1') }} -- exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql index 962cb57a8666..c645234a8db9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -14,4 +18,6 @@ select tmp.* from {{ ref('exchange_rate_ab2') }} tmp -- exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql index 1041493c6c67..67bc11204a5c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, @@ -7,7 +11,10 @@ select {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_log_pos'], ['_ab_cdc_log_pos']) }} as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} as table_alias -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql index bccb189a9d7a..a58511ceb6b7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -7,7 +11,10 @@ select cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, cast(_ab_cdc_log_pos as {{ dbt_utils.type_float() }}) as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('pos_dedup_cdcx_ab1') }} -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index 0fd3196cea9f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'name', - '_ab_cdc_lsn', - '_ab_cdc_updated_at', - '_ab_cdc_deleted_at', - '_ab_cdc_log_pos', - ]) }} as _airbyte_pos_dedup_cdcx_hashid, - tmp.* -from {{ ref('pos_dedup_cdcx_ab2') }} tmp --- pos_dedup_cdcx - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql deleted file mode 100644 index 7c897124b395..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_pos_dedup_cdcx_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('pos_dedup_cdcx_ab3') }} tmp --- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..7fd3046082a7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,16 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..a04f92bdcd55 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,16 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as {{ dbt_utils.type_bigint() }}) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ ref('renamed_dedup_cdc_excluded_ab1') }} +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..d086aa78b7bc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,111 @@ +{{ config( + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('dedup_cdc_excluded_ab3') }} + -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('dedup_cdc_excluded_ab3') }} + -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}) + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..d406c4709941 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,121 @@ +{{ config( + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('dedup_exchange_rate_ab3') }} + -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + 'currency', + 'nzd', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('dedup_exchange_rate_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('dedup_exchange_rate_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('dedup_exchange_rate_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('dedup_exchange_rate_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('dedup_exchange_rate_ab3') }} + -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + 'currency', + 'nzd', + ]) }} as _airbyte_unique_key, + id, + currency, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + {{ adapter.quote('date') }} as _airbyte_start_at, + lag({{ adapter.quote('date') }}) over ( + partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) + order by + {{ adapter.quote('date') }} desc, + {{ adapter.quote('date') }} desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag({{ adapter.quote('date') }}) over ( + partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) + order by + {{ adapter.quote('date') }} desc, + {{ adapter.quote('date') }} desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_exchange_rate_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + currency, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..67ab2bb4e05a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,103 @@ +{{ config( + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('renamed_dedup_cdc_excluded_ab3') }} + -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('renamed_dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('renamed_dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('renamed_dedup_cdc_excluded_ab3') }} + -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_renamed_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql new file mode 100644 index 000000000000..086676f17371 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -0,0 +1,23 @@ +{{ config( + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from {{ ref('dedup_cdc_excluded_scd') }} +-- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..06a37a760378 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,26 @@ +{{ config( + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + id, + currency, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from {{ ref('dedup_exchange_rate_scd') }} +-- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..980126a4146c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql @@ -0,0 +1,24 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + id, + currency, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from {{ ref('exchange_rate_ab3') }} +-- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..8a8ff85f5902 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,19 @@ +{{ config( + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from {{ ref('renamed_dedup_cdc_excluded_scd') }} +-- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..4d32358f6f28 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,113 @@ +{{ config( + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('pos_dedup_cdcx_ab3') }} + -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('pos_dedup_cdcx_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('pos_dedup_cdcx_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('pos_dedup_cdcx_ab3') }} + -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_log_pos as {{ dbt_utils.type_string() }}) + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at', '_ab_cdc_log_pos' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index 2e53ecaf2bee..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - name, - {{ adapter.quote('column`_\'with""_quotes') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from {{ ref('dedup_cdc_excluded_scd') }} --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -where _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index f6b8e8c4f3af..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - {{ adapter.quote('column`_\'with""_quotes') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at desc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at desc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from {{ ref('dedup_cdc_excluded_ab4') }} --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql deleted file mode 100644 index f10a9446d7b8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - currency, - {{ adapter.quote('date') }}, - timestamp_col, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from {{ ref('dedup_exchange_rate_scd') }} --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -where _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql deleted file mode 100644 index 77ef36c1fe48..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql +++ /dev/null @@ -1,26 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - currency, - {{ adapter.quote('date') }}, - timestamp_col, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, - {{ adapter.quote('date') }} as _airbyte_start_at, - lag({{ adapter.quote('date') }}) over ( - partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) - order by {{ adapter.quote('date') }} desc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag({{ adapter.quote('date') }}) over ( - partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) - order by {{ adapter.quote('date') }} desc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from {{ ref('dedup_exchange_rate_ab4') }} --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql deleted file mode 100644 index 147399e6739d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - currency, - {{ adapter.quote('date') }}, - timestamp_col, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, - _airbyte_emitted_at, - _airbyte_exchange_rate_hashid -from {{ ref('exchange_rate_ab3') }} --- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql index ed02a5d04940..fc341acc8a63 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -1,15 +1,23 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select + _airbyte_unique_key, id, name, _ab_cdc_lsn, _ab_cdc_updated_at, _ab_cdc_deleted_at, _ab_cdc_log_pos, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_pos_dedup_cdcx_hashid from {{ ref('pos_dedup_cdcx_scd') }} -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index a5f9705f3642..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at desc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at desc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid -from {{ ref('pos_dedup_cdcx_ab4') }} --- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..dc6238af14f4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,20 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + 'name', + '_ab_cdc_lsn', + '_ab_cdc_updated_at', + '_ab_cdc_deleted_at', + ]) }} as _airbyte_dedup_cdc_excluded_hashid, + tmp.* +from {{ ref('dedup_cdc_excluded_ab2') }} tmp +-- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..9221c1833031 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,23 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + 'currency', + adapter.quote('date'), + 'timestamp_col', + adapter.quote('HKD@spéçiäl & characters'), + 'hkd_special___characters', + 'nzd', + 'usd', + ]) }} as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from {{ ref('dedup_exchange_rate_ab2') }} tmp +-- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql new file mode 100644 index 000000000000..7dd3908bac88 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -0,0 +1,20 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + 'name', + '_ab_cdc_lsn', + '_ab_cdc_updated_at', + '_ab_cdc_deleted_at', + '_ab_cdc_log_pos', + ]) }} as _airbyte_pos_dedup_cdcx_hashid, + tmp.* +from {{ ref('pos_dedup_cdcx_ab2') }} tmp +-- pos_dedup_cdcx +where 1 = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..f5ce3e6ba182 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,16 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from {{ ref('renamed_dedup_cdc_excluded_ab2') }} tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/sources.yml index c085ca8ccc25..45c338b893ca 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/sources.yml @@ -10,3 +10,4 @@ sources: - name: _airbyte_raw_dedup_exchange_rate - name: _airbyte_raw_exchange_rate - name: _airbyte_raw_pos_dedup_cdcx + - name: _airbyte_raw_renamed_dedup_cdc_excluded diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..7ca4f64da70a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."dedup_cdc_excluded_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."name"'') as name, + json_value(_airbyte_data, ''$."_ab_cdc_lsn"'') as _ab_cdc_lsn, + json_value(_airbyte_data, ''$."_ab_cdc_updated_at"'') as _ab_cdc_updated_at, + json_value(_airbyte_data, ''$."_ab_cdc_deleted_at"'') as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias +-- dedup_cdc_excluded +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..071d2319c025 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -0,0 +1,28 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."dedup_cdc_excluded_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(name as + VARCHAR(max)) as name, + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."dedup_cdc_excluded_ab1" +-- dedup_cdc_excluded +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql new file mode 100644 index 000000000000..62877d8eede6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -0,0 +1,22 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."dedup_exchange_rate_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."currency"'') as currency, + json_value(_airbyte_data, ''$."date"'') as "date", + json_value(_airbyte_data, ''$."timestamp_col"'') as timestamp_col, + json_value(_airbyte_data, ''$."HKD@spéçiäl & characters"'') as "HKD@spéçiäl & characters", + json_value(_airbyte_data, ''$."HKD_special___characters"'') as hkd_special___characters, + json_value(_airbyte_data, ''$."NZD"'') as nzd, + json_value(_airbyte_data, ''$."USD"'') as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql new file mode 100644 index 000000000000..b871d6e41b0b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -0,0 +1,32 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."dedup_exchange_rate_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(currency as + VARCHAR(max)) as currency, + try_parse(nullif("date", '''') as date) as "date", + try_parse(nullif(timestamp_col, '''') as datetime) as timestamp_col, + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(hkd_special___characters as + VARCHAR(max)) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."dedup_exchange_rate_ab1" +-- dedup_exchange_rate +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql new file mode 100644 index 000000000000..31eb5bb200ab --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -0,0 +1,22 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."exchange_rate_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."currency"'') as currency, + json_value(_airbyte_data, ''$."date"'') as "date", + json_value(_airbyte_data, ''$."timestamp_col"'') as timestamp_col, + json_value(_airbyte_data, ''$."HKD@spéçiäl & characters"'') as "HKD@spéçiäl & characters", + json_value(_airbyte_data, ''$."HKD_special___characters"'') as hkd_special___characters, + json_value(_airbyte_data, ''$."NZD"'') as nzd, + json_value(_airbyte_data, ''$."USD"'') as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql new file mode 100644 index 000000000000..bf6e40566af3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -0,0 +1,32 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."exchange_rate_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(currency as + VARCHAR(max)) as currency, + try_parse(nullif("date", '''') as date) as "date", + try_parse(nullif(timestamp_col, '''') as datetime) as timestamp_col, + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(hkd_special___characters as + VARCHAR(max)) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."exchange_rate_ab1" +-- exchange_rate +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql new file mode 100644 index 000000000000..13c53bf0f858 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -0,0 +1,26 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."exchange_rate_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as + VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as + VARCHAR(max)), ''''), ''-'', coalesce(cast(timestamp_col as + VARCHAR(max)), ''''), ''-'', coalesce(cast("HKD@spéçiäl & characters" as + VARCHAR(max)), ''''), ''-'', coalesce(cast(hkd_special___characters as + VARCHAR(max)), ''''), ''-'', coalesce(cast(nzd as + VARCHAR(max)), ''''), ''-'', coalesce(cast(usd as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_exchange_rate_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."exchange_rate_ab2" tmp +-- exchange_rate +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql new file mode 100644 index 000000000000..d210ebebcdfc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."pos_dedup_cdcx_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."name"'') as name, + json_value(_airbyte_data, ''$."_ab_cdc_lsn"'') as _ab_cdc_lsn, + json_value(_airbyte_data, ''$."_ab_cdc_updated_at"'') as _ab_cdc_updated_at, + json_value(_airbyte_data, ''$."_ab_cdc_deleted_at"'') as _ab_cdc_deleted_at, + json_value(_airbyte_data, ''$."_ab_cdc_log_pos"'') as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias +-- pos_dedup_cdcx +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql new file mode 100644 index 000000000000..d23785697728 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -0,0 +1,30 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."pos_dedup_cdcx_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(name as + VARCHAR(max)) as name, + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + cast(_ab_cdc_log_pos as + float +) as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."pos_dedup_cdcx_ab1" +-- pos_dedup_cdcx +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..7c05f028e95e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,15 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..362c8ff7d323 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,17 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1" +-- renamed_dedup_cdc_excluded +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..a9f7cffc2289 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,14 @@ + + delete + from "test_normalization".test_normalization."dedup_cdc_excluded_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "test_normalization".test_normalization."#dedup_cdc_excluded_scd__dbt_tmp" + ); + + insert into "test_normalization".test_normalization."dedup_cdc_excluded_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid" + from "test_normalization".test_normalization."#dedup_cdc_excluded_scd__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..6af21568463d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,14 @@ + + delete + from "test_normalization".test_normalization."dedup_exchange_rate_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "test_normalization".test_normalization."#dedup_exchange_rate_scd__dbt_tmp" + ); + + insert into "test_normalization".test_normalization."dedup_exchange_rate_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid" + from "test_normalization".test_normalization."#dedup_exchange_rate_scd__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..5d17dd49a167 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,14 @@ + + delete + from "test_normalization".test_normalization."renamed_dedup_cdc_excluded_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "test_normalization".test_normalization."#renamed_dedup_cdc_excluded_scd__dbt_tmp" + ); + + insert into "test_normalization".test_normalization."renamed_dedup_cdc_excluded_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid" + from "test_normalization".test_normalization."#renamed_dedup_cdc_excluded_scd__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql new file mode 100644 index 000000000000..0f76179fe523 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -0,0 +1,14 @@ + + delete + from "test_normalization".test_normalization."dedup_cdc_excluded" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "test_normalization".test_normalization."#dedup_cdc_excluded__dbt_tmp" + ); + + insert into "test_normalization".test_normalization."dedup_cdc_excluded" ("_airbyte_unique_key", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid") + ( + select "_airbyte_unique_key", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid" + from "test_normalization".test_normalization."#dedup_cdc_excluded__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..c723e5310501 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,14 @@ + + delete + from "test_normalization".test_normalization."dedup_exchange_rate" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "test_normalization".test_normalization."#dedup_exchange_rate__dbt_tmp" + ); + + insert into "test_normalization".test_normalization."dedup_exchange_rate" ("_airbyte_unique_key", "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") + ( + select "_airbyte_unique_key", "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid" + from "test_normalization".test_normalization."#dedup_exchange_rate__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..993b2a0893ae --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -0,0 +1,14 @@ + + delete + from "test_normalization".test_normalization."exchange_rate" + where (_airbyte_ab_id) in ( + select (_airbyte_ab_id) + from "test_normalization".test_normalization."#exchange_rate__dbt_tmp" + ); + + insert into "test_normalization".test_normalization."exchange_rate" ("id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid") + ( + select "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid" + from "test_normalization".test_normalization."#exchange_rate__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..5b42db702fd8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,14 @@ + + delete + from "test_normalization".test_normalization."renamed_dedup_cdc_excluded" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "test_normalization".test_normalization."#renamed_dedup_cdc_excluded__dbt_tmp" + ); + + insert into "test_normalization".test_normalization."renamed_dedup_cdc_excluded" ("_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid") + ( + select "_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid" + from "test_normalization".test_normalization."#renamed_dedup_cdc_excluded__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..749435773423 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,134 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."pos_dedup_cdcx_scd__dbt_tmp"','U') is not null + begin + drop table test_normalization."pos_dedup_cdcx_scd__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" as + +with + +input_data as ( + select * + from "test_normalization"._airbyte_test_normalization."pos_dedup_cdcx_ab3" + -- pos_dedup_cdcx from "test_normalization".test_normalization._airbyte_raw_pos_dedup_cdcx +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as + VARCHAR(max)), cast(_ab_cdc_updated_at as + VARCHAR(max)), cast(_ab_cdc_log_pos as + VARCHAR(max)) + order by _airbyte_ab_id + ) as _airbyte_row_num, + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_unique_key as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_start_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_emitted_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_deleted_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_updated_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_log_pos as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."pos_dedup_cdcx_scd__dbt_tmp" FROM + "test_normalization".test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_pos_dedup_cdcx_scd__dbt_tmp_cci' + AND object_id=object_id('test_normalization_pos_dedup_cdcx_scd__dbt_tmp') + ) + DROP index test_normalization.pos_dedup_cdcx_scd__dbt_tmp.test_normalization_pos_dedup_cdcx_scd__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_pos_dedup_cdcx_scd__dbt_tmp_cci + ON test_normalization.pos_dedup_cdcx_scd__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql new file mode 100644 index 000000000000..846de1108761 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -0,0 +1,64 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."pos_dedup_cdcx__dbt_tmp"','U') is not null + begin + drop table test_normalization."pos_dedup_cdcx__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view" as + +-- Final base SQL model +select + _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from "test_normalization".test_normalization."pos_dedup_cdcx_scd" +-- pos_dedup_cdcx from "test_normalization".test_normalization._airbyte_raw_pos_dedup_cdcx +where 1 = 1 +and _airbyte_active_row = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."pos_dedup_cdcx__dbt_tmp" FROM + "test_normalization".test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_pos_dedup_cdcx__dbt_tmp_cci' + AND object_id=object_id('test_normalization_pos_dedup_cdcx__dbt_tmp') + ) + DROP index test_normalization.pos_dedup_cdcx__dbt_tmp.test_normalization_pos_dedup_cdcx__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_pos_dedup_cdcx__dbt_tmp_cci + ON test_normalization.pos_dedup_cdcx__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..71b4b90ce304 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,23 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."dedup_cdc_excluded_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(name as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_lsn as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_updated_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_deleted_at as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_dedup_cdc_excluded_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."dedup_cdc_excluded_ab2" tmp +-- dedup_cdc_excluded +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..021c2a7b88f4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,26 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."dedup_exchange_rate_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as + VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as + VARCHAR(max)), ''''), ''-'', coalesce(cast(timestamp_col as + VARCHAR(max)), ''''), ''-'', coalesce(cast("HKD@spéçiäl & characters" as + VARCHAR(max)), ''''), ''-'', coalesce(cast(hkd_special___characters as + VARCHAR(max)), ''''), ''-'', coalesce(cast(nzd as + VARCHAR(max)), ''''), ''-'', coalesce(cast(usd as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."dedup_exchange_rate_ab2" tmp +-- dedup_exchange_rate +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql new file mode 100644 index 000000000000..fe243c2ddc74 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -0,0 +1,23 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."pos_dedup_cdcx_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(name as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_lsn as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_updated_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_deleted_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_log_pos as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_pos_dedup_cdcx_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."pos_dedup_cdcx_ab2" tmp +-- pos_dedup_cdcx +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..6781d88c4b0f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2" tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql deleted file mode 100644 index 16064ab14c2f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_array_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_extract(_airbyte_data, - '$."conflict_stream_array"') as conflict_stream_array, - _airbyte_emitted_at -from test_normalization._airbyte_raw_conflict_stream_array as table_alias --- conflict_stream_array - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql deleted file mode 100644 index 2ab090a7e559..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_array_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as char) as id, - conflict_stream_array, - _airbyte_emitted_at -from _airbyte_test_normalization.`conflict_stream_array_ab1` --- conflict_stream_array - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql deleted file mode 100644 index c4a87eaa9ec9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ /dev/null @@ -1,10 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_array_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(conflict_stream_array as char), '')) as char)) as _airbyte_conflict_stream_array_hashid, - tmp.* -from _airbyte_test_normalization.`conflict_stream_array_ab2` tmp --- conflict_stream_array - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql deleted file mode 100644 index e5689bca73bd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql +++ /dev/null @@ -1,15 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_hashid, - - json_extract(table_alias.conflict_stream_name, - '$."conflict_stream_name"') - as conflict_stream_name, - _airbyte_emitted_at -from test_normalization.`conflict_stream_name` as table_alias -where conflict_stream_name is not null --- conflict_stream_name at conflict_stream_name/conflict_stream_name - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql deleted file mode 100644 index 7a25f59f18f0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_hashid, - cast(conflict_stream_name as json) as conflict_stream_name, - _airbyte_emitted_at -from _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab1` --- conflict_stream_name at conflict_stream_name/conflict_stream_name - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql deleted file mode 100644 index c77e0c72a4f7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql +++ /dev/null @@ -1,10 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(_airbyte_conflict_stream_name_hashid as char), ''), '-', coalesce(cast(conflict_stream_name as char), '')) as char)) as _airbyte_conflict_stream_name_2_hashid, - tmp.* -from _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab2` tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql deleted file mode 100644 index d786dfa4a89e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_2_hashid, - json_value(conflict_stream_name, - '$."groups"') as `groups`, - _airbyte_emitted_at -from test_normalization.`conflict_stream_name_conflict_stream_name` as table_alias -where conflict_stream_name is not null --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql deleted file mode 100644 index 3393dde64ef8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_2_hashid, - cast(`groups` as char) as `groups`, - _airbyte_emitted_at -from _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab1` --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql deleted file mode 100644 index 24929b82dca9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql +++ /dev/null @@ -1,10 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as char), ''), '-', coalesce(cast(`groups` as char), '')) as char)) as _airbyte_conflict_stream_name_3_hashid, - tmp.* -from _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab2` tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql deleted file mode 100644 index a575697fcb36..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ /dev/null @@ -1,15 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_name_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - - json_extract(table_alias._airbyte_data, - '$."conflict_stream_name"') - as conflict_stream_name, - _airbyte_emitted_at -from test_normalization._airbyte_raw_conflict_stream_name as table_alias --- conflict_stream_name - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql deleted file mode 100644 index f9ad98a47215..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_name_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as char) as id, - cast(conflict_stream_name as json) as conflict_stream_name, - _airbyte_emitted_at -from _airbyte_test_normalization.`conflict_stream_name_ab1` --- conflict_stream_name - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql deleted file mode 100644 index 65dd3434a822..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ /dev/null @@ -1,10 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_name_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(conflict_stream_name as char), '')) as char)) as _airbyte_conflict_stream_name_hashid, - tmp.* -from _airbyte_test_normalization.`conflict_stream_name_ab2` tmp --- conflict_stream_name - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql deleted file mode 100644 index c97c708d399b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_scalar_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_value(_airbyte_data, - '$."conflict_stream_scalar"') as conflict_stream_scalar, - _airbyte_emitted_at -from test_normalization._airbyte_raw_conflict_stream_scalar as table_alias --- conflict_stream_scalar - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql deleted file mode 100644 index fc08f53590d0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_scalar_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as char) as id, - cast(conflict_stream_scalar as - signed -) as conflict_stream_scalar, - _airbyte_emitted_at -from _airbyte_test_normalization.`conflict_stream_scalar_ab1` --- conflict_stream_scalar - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql deleted file mode 100644 index 88e6caaca33d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ /dev/null @@ -1,10 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_scalar_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(conflict_stream_scalar as char), '')) as char)) as _airbyte_conflict_stream_scalar_hashid, - tmp.* -from _airbyte_test_normalization.`conflict_stream_scalar_ab2` tmp --- conflict_stream_scalar - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql deleted file mode 100644 index 81d223bc373c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_value(_airbyte_data, - '$."date"') as `date`, - - json_extract(table_alias._airbyte_data, - '$."partition"') - as `partition`, - _airbyte_emitted_at -from test_normalization._airbyte_raw_nested_s__lting_into_long_names as table_alias --- nested_stream_with_co__lting_into_long_names - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql deleted file mode 100644 index 23b64ee8f8e5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql +++ /dev/null @@ -1,12 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as char) as id, - cast(`date` as char) as `date`, - cast(`partition` as json) as `partition`, - _airbyte_emitted_at -from _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab1` --- nested_stream_with_co__lting_into_long_names - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql deleted file mode 100644 index 1b0b96e37ab7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql +++ /dev/null @@ -1,10 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(`partition` as char), '')) as char)) as _airbyte_nested_strea__nto_long_names_hashid, - tmp.* -from _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab2` tmp --- nested_stream_with_co__lting_into_long_names - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab4.sql deleted file mode 100644 index fe5092015be9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab4.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab4__dbt_tmp` as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_nested_strea__nto_long_names_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab3` tmp --- nested_stream_with_co__lting_into_long_names from test_normalization._airbyte_raw_nested_s__lting_into_long_names - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql deleted file mode 100644 index 5ba25fb20ebc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_nested_strea__nto_long_names_hashid, - json_extract(`partition`, - '$."double_array_data"') as double_array_data, - json_extract(`partition`, - '$."DATA"') as `DATA`, - json_extract(`partition`, - '$."column___with__quotes"') as `column__'with"_quotes`, - _airbyte_emitted_at -from test_normalization.`nested_stream_with_co__lting_into_long_names` as table_alias -where `partition` is not null --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql deleted file mode 100644 index 56a4acbc626d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_nested_strea__nto_long_names_hashid, - double_array_data, - `DATA`, - `column__'with"_quotes`, - _airbyte_emitted_at -from _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab1` --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql deleted file mode 100644 index 5b0ac76d32b3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql +++ /dev/null @@ -1,10 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(_airbyte_nested_strea__nto_long_names_hashid as char), ''), '-', coalesce(cast(double_array_data as char), ''), '-', coalesce(cast(`DATA` as char), ''), '-', coalesce(cast(`column__'with"_quotes` as char), '')) as char)) as _airbyte_partition_hashid, - tmp.* -from _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab2` tmp --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql deleted file mode 100644 index 6bb578315008..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql +++ /dev/null @@ -1,59 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -with numbers as ( - - - - - with p as ( - select 0 as generated_number union all select 1 - ), unioned as ( - - select - - - p0.generated_number * pow(2, 0) - - - + 1 - as generated_number - - from - - - p as p0 - - - - ) - - select * - from unioned - where generated_number <= 2 - order by generated_number - - - ), - joined as ( - select - _airbyte_partition_hashid as _airbyte_hashid, - - json_extract(double_array_data, concat("$[", numbers.generated_number - 1, "][0]")) as _airbyte_nested_data - from test_normalization.`nested_stream_with_co___long_names_partition` - cross join numbers - -- only generate the number of records in the cross join that corresponds - -- to the number of items in nested_stream_with_co___long_names_partition.double_array_data - where numbers.generated_number <= json_length(double_array_data) - ) -select - _airbyte_partition_hashid, - json_value(_airbyte_nested_data, - '$."id"') as id, - _airbyte_emitted_at -from test_normalization.`nested_stream_with_co___long_names_partition` as table_alias -left join joined on _airbyte_partition_hashid = joined._airbyte_hashid -where double_array_data is not null --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql deleted file mode 100644 index 43a2faa29900..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(id as char) as id, - _airbyte_emitted_at -from _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab1` --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql deleted file mode 100644 index 39db1c9a56a1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql +++ /dev/null @@ -1,10 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as char), ''), '-', coalesce(cast(id as char), '')) as char)) as _airbyte_double_array_data_hashid, - tmp.* -from _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab2` tmp --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql deleted file mode 100644 index a532d3e22f8f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql +++ /dev/null @@ -1,59 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -with numbers as ( - - - - - with p as ( - select 0 as generated_number union all select 1 - ), unioned as ( - - select - - - p0.generated_number * pow(2, 0) - - - + 1 - as generated_number - - from - - - p as p0 - - - - ) - - select * - from unioned - where generated_number <= 1 - order by generated_number - - - ), - joined as ( - select - _airbyte_partition_hashid as _airbyte_hashid, - - json_extract(`DATA`, concat("$[", numbers.generated_number - 1, "][0]")) as _airbyte_nested_data - from test_normalization.`nested_stream_with_co___long_names_partition` - cross join numbers - -- only generate the number of records in the cross join that corresponds - -- to the number of items in nested_stream_with_co___long_names_partition.`DATA` - where numbers.generated_number <= json_length(`DATA`) - ) -select - _airbyte_partition_hashid, - json_value(_airbyte_nested_data, - '$."currency"') as currency, - _airbyte_emitted_at -from test_normalization.`nested_stream_with_co___long_names_partition` as table_alias -left join joined on _airbyte_partition_hashid = joined._airbyte_hashid -where `DATA` is not null --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql deleted file mode 100644 index 4c7968e3c11d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as char) as currency, - _airbyte_emitted_at -from _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab1` --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql deleted file mode 100644 index 0bd43b99279c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql +++ /dev/null @@ -1,10 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as char), ''), '-', coalesce(cast(currency as char), '')) as char)) as _airbyte_data_hashid, - tmp.* -from _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab2` tmp --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql deleted file mode 100644 index 4f3b7632f1b1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql +++ /dev/null @@ -1,59 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -with numbers as ( - - - - - with p as ( - select 0 as generated_number union all select 1 - ), unioned as ( - - select - - - p0.generated_number * pow(2, 0) - - - + 1 - as generated_number - - from - - - p as p0 - - - - ) - - select * - from unioned - where generated_number <= 1 - order by generated_number - - - ), - joined as ( - select - _airbyte_partition_hashid as _airbyte_hashid, - - json_extract(`column__'with"_quotes`, concat("$[", numbers.generated_number - 1, "][0]")) as _airbyte_nested_data - from test_normalization.`nested_stream_with_co___long_names_partition` - cross join numbers - -- only generate the number of records in the cross join that corresponds - -- to the number of items in nested_stream_with_co___long_names_partition.`column__'with"_quotes` - where numbers.generated_number <= json_length(`column__'with"_quotes`) - ) -select - _airbyte_partition_hashid, - json_value(_airbyte_nested_data, - '$."currency"') as currency, - _airbyte_emitted_at -from test_normalization.`nested_stream_with_co___long_names_partition` as table_alias -left join joined on _airbyte_partition_hashid = joined._airbyte_hashid -where `column__'with"_quotes` is not null --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql deleted file mode 100644 index 097f8b5b835e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as char) as currency, - _airbyte_emitted_at -from _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab1` --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql deleted file mode 100644 index e9ecd923a8f4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql +++ /dev/null @@ -1,10 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as char), ''), '-', coalesce(cast(currency as char), '')) as char)) as _airbyte_column___with__quotes_hashid, - tmp.* -from _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab2` tmp --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql deleted file mode 100644 index 590bb28948ca..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_value(_airbyte_data, - '$."date"') as `date`, - _airbyte_emitted_at -from test_normalization._airbyte_raw_non_nest__lting_into_long_names as table_alias --- non_nested_stream_wit__lting_into_long_names - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql deleted file mode 100644 index 8c37318b3d29..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as char) as id, - cast(`date` as char) as `date`, - _airbyte_emitted_at -from _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab1` --- non_nested_stream_wit__lting_into_long_names - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql deleted file mode 100644 index 4d0c9bc16263..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql +++ /dev/null @@ -1,10 +0,0 @@ - - create view _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`date` as char), '')) as char)) as _airbyte_non_nested_s__nto_long_names_hashid, - tmp.* -from _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab2` tmp --- non_nested_stream_wit__lting_into_long_names - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab1.sql deleted file mode 100644 index 6e843ca7a657..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view _airbyte_test_normalization.`unnest_alias_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_extract(_airbyte_data, - '$."children"') as children, - _airbyte_emitted_at -from test_normalization._airbyte_raw_unnest_alias as table_alias --- unnest_alias - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab2.sql deleted file mode 100644 index a7d929e77df0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view _airbyte_test_normalization.`unnest_alias_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - signed -) as id, - children, - _airbyte_emitted_at -from _airbyte_test_normalization.`unnest_alias_ab1` --- unnest_alias - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab3.sql deleted file mode 100644 index 4d48087571c3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ /dev/null @@ -1,10 +0,0 @@ - - create view _airbyte_test_normalization.`unnest_alias_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(children as char), '')) as char)) as _airbyte_unnest_alias_hashid, - tmp.* -from _airbyte_test_normalization.`unnest_alias_ab2` tmp --- unnest_alias - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql deleted file mode 100644 index 0b2b2a86bd2d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ /dev/null @@ -1,63 +0,0 @@ - - create view _airbyte_test_normalization.`unnest_alias_children_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -with numbers as ( - - - - - with p as ( - select 0 as generated_number union all select 1 - ), unioned as ( - - select - - - p0.generated_number * pow(2, 0) - - - + 1 - as generated_number - - from - - - p as p0 - - - - ) - - select * - from unioned - where generated_number <= 2 - order by generated_number - - - ), - joined as ( - select - _airbyte_unnest_alias_hashid as _airbyte_hashid, - - json_extract(children, concat("$[", numbers.generated_number - 1, "][0]")) as _airbyte_nested_data - from test_normalization.`unnest_alias` - cross join numbers - -- only generate the number of records in the cross join that corresponds - -- to the number of items in unnest_alias.children - where numbers.generated_number <= json_length(children) - ) -select - _airbyte_unnest_alias_hashid, - json_value(_airbyte_nested_data, - '$."ab_id"') as ab_id, - - json_extract(_airbyte_nested_data, - '$."owner"') - as `owner`, - _airbyte_emitted_at -from test_normalization.`unnest_alias` as table_alias -left join joined on _airbyte_unnest_alias_hashid = joined._airbyte_hashid -where children is not null --- children at unnest_alias/children - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql deleted file mode 100644 index 3799355bfffe..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create view _airbyte_test_normalization.`unnest_alias_children_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_unnest_alias_hashid, - cast(ab_id as - signed -) as ab_id, - cast(`owner` as json) as `owner`, - _airbyte_emitted_at -from _airbyte_test_normalization.`unnest_alias_children_ab1` --- children at unnest_alias/children - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql deleted file mode 100644 index e99957b0b0e5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ /dev/null @@ -1,10 +0,0 @@ - - create view _airbyte_test_normalization.`unnest_alias_children_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(_airbyte_unnest_alias_hashid as char), ''), '-', coalesce(cast(ab_id as char), ''), '-', coalesce(cast(`owner` as char), '')) as char)) as _airbyte_children_hashid, - tmp.* -from _airbyte_test_normalization.`unnest_alias_children_ab2` tmp --- children at unnest_alias/children - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql deleted file mode 100644 index a8080abb06b5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view _airbyte_test_normalization.`unnest_alias_children_owner_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_children_hashid, - json_value(`owner`, - '$."owner_id"') as owner_id, - _airbyte_emitted_at -from test_normalization.`unnest_alias_children` as table_alias -where `owner` is not null --- owner at unnest_alias/children/owner - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql deleted file mode 100644 index e96688a1c6cc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view _airbyte_test_normalization.`unnest_alias_children_owner_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_children_hashid, - cast(owner_id as - signed -) as owner_id, - _airbyte_emitted_at -from _airbyte_test_normalization.`unnest_alias_children_owner_ab1` --- owner at unnest_alias/children/owner - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql deleted file mode 100644 index ac7aeed08683..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ /dev/null @@ -1,10 +0,0 @@ - - create view _airbyte_test_normalization.`unnest_alias_children_owner_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(_airbyte_children_hashid as char), ''), '-', coalesce(cast(owner_id as char), '')) as char)) as _airbyte_owner_hashid, - tmp.* -from _airbyte_test_normalization.`unnest_alias_children_owner_ab2` tmp --- owner at unnest_alias/children/owner - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql deleted file mode 100644 index 52aa78db11bd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_value(_airbyte_data, - '$."date"') as `date`, - _airbyte_emitted_at -from test_normalization_namespace._airbyte_raw_simple_s__lting_into_long_names as table_alias --- simple_stream_with_na__lting_into_long_names - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql deleted file mode 100644 index 679f88767d21..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as char) as id, - cast(`date` as char) as `date`, - _airbyte_emitted_at -from _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab1` --- simple_stream_with_na__lting_into_long_names - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql deleted file mode 100644 index b2c0230b903c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql +++ /dev/null @@ -1,10 +0,0 @@ - - create view _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`date` as char), '')) as char)) as _airbyte_simple_strea__nto_long_names_hashid, - tmp.* -from _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab2` tmp --- simple_stream_with_na__lting_into_long_names - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql deleted file mode 100644 index 2c1eed7331a4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create table - test_normalization.`conflict_stream_array__dbt_tmp` - as ( - --- Final base SQL model -select - id, - conflict_stream_array, - _airbyte_emitted_at, - _airbyte_conflict_stream_array_hashid -from _airbyte_test_normalization.`conflict_stream_array_ab3` --- conflict_stream_array from test_normalization._airbyte_raw_conflict_stream_array - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql deleted file mode 100644 index d341fcab11c6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create table - test_normalization.`conflict_stream_name__dbt_tmp` - as ( - --- Final base SQL model -select - id, - conflict_stream_name, - _airbyte_emitted_at, - _airbyte_conflict_stream_name_hashid -from _airbyte_test_normalization.`conflict_stream_name_ab3` --- conflict_stream_name from test_normalization._airbyte_raw_conflict_stream_name - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql deleted file mode 100644 index c55a5e3a62ed..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create table - test_normalization.`conflict_stream_name____conflict_stream_name__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_conflict_stream_name_2_hashid, - `groups`, - _airbyte_emitted_at, - _airbyte_conflict_stream_name_3_hashid -from _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab3` --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from test_normalization.`conflict_stream_name_conflict_stream_name` - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql deleted file mode 100644 index 8cf88cc99275..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create table - test_normalization.`conflict_stream_name_conflict_stream_name__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_conflict_stream_name_hashid, - conflict_stream_name, - _airbyte_emitted_at, - _airbyte_conflict_stream_name_2_hashid -from _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab3` --- conflict_stream_name at conflict_stream_name/conflict_stream_name from test_normalization.`conflict_stream_name` - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql deleted file mode 100644 index aad49db6aa81..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create table - test_normalization.`conflict_stream_scalar__dbt_tmp` - as ( - --- Final base SQL model -select - id, - conflict_stream_scalar, - _airbyte_emitted_at, - _airbyte_conflict_stream_scalar_hashid -from _airbyte_test_normalization.`conflict_stream_scalar_ab3` --- conflict_stream_scalar from test_normalization._airbyte_raw_conflict_stream_scalar - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql deleted file mode 100644 index 88904c5291b2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql +++ /dev/null @@ -1,26 +0,0 @@ - - - create table - test_normalization.`nested_stream_with_co_1g_into_long_names_scd__dbt_tmp` - as ( - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - `date`, - `partition`, - `date` as _airbyte_start_at, - lag(`date`) over ( - partition by id - order by `date` is null asc, `date` desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(`date`) over ( - partition by id - order by `date` is null asc, `date` desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_nested_strea__nto_long_names_hashid -from _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab4` --- nested_stream_with_co__lting_into_long_names from test_normalization._airbyte_raw_nested_s__lting_into_long_names -where _airbyte_row_num = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql deleted file mode 100644 index a796e4bec478..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql +++ /dev/null @@ -1,17 +0,0 @@ - - - create table - test_normalization.`nested_stream_with_co___long_names_partition__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_nested_strea__nto_long_names_hashid, - double_array_data, - `DATA`, - `column__'with"_quotes`, - _airbyte_emitted_at, - _airbyte_partition_hashid -from _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab3` --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from test_normalization.`nested_stream_with_co__lting_into_long_names` - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql deleted file mode 100644 index 829800f026b8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create table - test_normalization.`nested_stream_with_co___names_partition_data__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_emitted_at, - _airbyte_data_hashid -from _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab3` --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from test_normalization.`nested_stream_with_co___long_names_partition` - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql deleted file mode 100644 index c65521b6b2c2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create table - test_normalization.`nested_stream_with_co__column___with__quotes__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_emitted_at, - _airbyte_column___with__quotes_hashid -from _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab3` --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from test_normalization.`nested_stream_with_co___long_names_partition` - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql deleted file mode 100644 index e0795e0c729c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create table - test_normalization.`nested_stream_with_co__ion_double_array_data__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_partition_hashid, - id, - _airbyte_emitted_at, - _airbyte_double_array_data_hashid -from _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab3` --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from test_normalization.`nested_stream_with_co___long_names_partition` - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql deleted file mode 100644 index a2c64e0181af..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql +++ /dev/null @@ -1,17 +0,0 @@ - - - create table - test_normalization.`nested_stream_with_co__lting_into_long_names__dbt_tmp` - as ( - --- Final base SQL model -select - id, - `date`, - `partition`, - _airbyte_emitted_at, - _airbyte_nested_strea__nto_long_names_hashid -from test_normalization.`nested_stream_with_co_1g_into_long_names_scd` --- nested_stream_with_co__lting_into_long_names from test_normalization._airbyte_raw_nested_s__lting_into_long_names -where _airbyte_active_row = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql deleted file mode 100644 index d80970404ecf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create table - test_normalization.`non_nested_stream_wit__lting_into_long_names__dbt_tmp` - as ( - --- Final base SQL model -select - id, - `date`, - _airbyte_emitted_at, - _airbyte_non_nested_s__nto_long_names_hashid -from _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab3` --- non_nested_stream_wit__lting_into_long_names from test_normalization._airbyte_raw_non_nest__lting_into_long_names - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql deleted file mode 100644 index cb9830768576..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create table - test_normalization.`unnest_alias__dbt_tmp` - as ( - --- Final base SQL model -select - id, - children, - _airbyte_emitted_at, - _airbyte_unnest_alias_hashid -from _airbyte_test_normalization.`unnest_alias_ab3` --- unnest_alias from test_normalization._airbyte_raw_unnest_alias - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql deleted file mode 100644 index 8d943344e1ea..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql +++ /dev/null @@ -1,16 +0,0 @@ - - - create table - test_normalization.`unnest_alias_children__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_unnest_alias_hashid, - ab_id, - `owner`, - _airbyte_emitted_at, - _airbyte_children_hashid -from _airbyte_test_normalization.`unnest_alias_children_ab3` --- children at unnest_alias/children from test_normalization.`unnest_alias` - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql deleted file mode 100644 index 74ae1e9027f2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create table - test_normalization.`unnest_alias_children_owner__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_children_hashid, - owner_id, - _airbyte_emitted_at, - _airbyte_owner_hashid -from _airbyte_test_normalization.`unnest_alias_children_owner_ab3` --- owner at unnest_alias/children/owner from test_normalization.`unnest_alias_children` - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql deleted file mode 100644 index c7220f05c387..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create table - test_normalization_namespace.`simple_stream_with_na__lting_into_long_names__dbt_tmp` - as ( - --- Final base SQL model -select - id, - `date`, - _airbyte_emitted_at, - _airbyte_simple_strea__nto_long_names_hashid -from _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab3` --- simple_stream_with_na__lting_into_long_names from test_normalization_namespace._airbyte_raw_simple_s__lting_into_long_names - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql new file mode 100644 index 000000000000..b08ffaf4f715 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`conflict_stream_array_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_extract(_airbyte_data, + '$."conflict_stream_array"') as conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_conflict_stream_array as table_alias +-- conflict_stream_array +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql new file mode 100644 index 000000000000..44b15b327edd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql @@ -0,0 +1,16 @@ + + create view _airbyte_test_normalization.`conflict_stream_array_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as char) as id, + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`conflict_stream_array_ab1` +-- conflict_stream_array +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql new file mode 100644 index 000000000000..857eb62be3f6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`conflict_stream_array_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(conflict_stream_array as char), '')) as char)) as _airbyte_conflict_stream_array_hashid, + tmp.* +from _airbyte_test_normalization.`conflict_stream_array_ab2` tmp +-- conflict_stream_array +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql new file mode 100644 index 000000000000..e3edc7461b2d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql @@ -0,0 +1,20 @@ + + create view _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_hashid, + + json_extract(table_alias.conflict_stream_name, + '$."conflict_stream_name"') + as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`conflict_stream_name` as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql new file mode 100644 index 000000000000..16def64e5011 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql @@ -0,0 +1,16 @@ + + create view _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_hashid, + cast(conflict_stream_name as json) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab1` +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql new file mode 100644 index 000000000000..99164479818a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_conflict_stream_name_hashid as char), ''), '-', coalesce(cast(conflict_stream_name as char), '')) as char)) as _airbyte_conflict_stream_name_2_hashid, + tmp.* +from _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab2` tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql new file mode 100644 index 000000000000..94cef187f111 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_2_hashid, + json_value(conflict_stream_name, + '$."groups"') as `groups`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`conflict_stream_name_conflict_stream_name` as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql new file mode 100644 index 000000000000..222256b4cc4e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql @@ -0,0 +1,16 @@ + + create view _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_2_hashid, + cast(`groups` as char) as `groups`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab1` +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql new file mode 100644 index 000000000000..2442380dac27 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as char), ''), '-', coalesce(cast(`groups` as char), '')) as char)) as _airbyte_conflict_stream_name_3_hashid, + tmp.* +from _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab2` tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..f0cefd4a6acf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql @@ -0,0 +1,20 @@ + + create view _airbyte_test_normalization.`conflict_stream_name_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + + json_extract(table_alias._airbyte_data, + '$."conflict_stream_name"') + as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_conflict_stream_name as table_alias +-- conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..b4f1e6a94ea6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql @@ -0,0 +1,16 @@ + + create view _airbyte_test_normalization.`conflict_stream_name_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as char) as id, + cast(conflict_stream_name as json) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`conflict_stream_name_ab1` +-- conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..8a7e62bbfafc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`conflict_stream_name_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(conflict_stream_name as char), '')) as char)) as _airbyte_conflict_stream_name_hashid, + tmp.* +from _airbyte_test_normalization.`conflict_stream_name_ab2` tmp +-- conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql new file mode 100644 index 000000000000..a30f57d7d4ed --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`conflict_stream_scalar_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."conflict_stream_scalar"') as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_conflict_stream_scalar as table_alias +-- conflict_stream_scalar +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql new file mode 100644 index 000000000000..5822b0221381 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`conflict_stream_scalar_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as char) as id, + cast(conflict_stream_scalar as + signed +) as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`conflict_stream_scalar_ab1` +-- conflict_stream_scalar +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql new file mode 100644 index 000000000000..6b31dc6f9b85 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`conflict_stream_scalar_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(conflict_stream_scalar as char), '')) as char)) as _airbyte_conflict_stream_scalar_hashid, + tmp.* +from _airbyte_test_normalization.`conflict_stream_scalar_ab2` tmp +-- conflict_stream_scalar +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql new file mode 100644 index 000000000000..2a4990a4233d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql @@ -0,0 +1,23 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."date"') as `date`, + + json_extract(table_alias._airbyte_data, + '$."partition"') + as `partition`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_nested_s__lting_into_long_names as table_alias +-- nested_stream_with_co__lting_into_long_names +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql new file mode 100644 index 000000000000..a5cd56efead8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as char) as id, + cast(`date` as char) as `date`, + cast(`partition` as json) as `partition`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab1` +-- nested_stream_with_co__lting_into_long_names +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql new file mode 100644 index 000000000000..328e7097da14 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql @@ -0,0 +1,22 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_nested_strea__nto_long_names_hashid, + json_extract(`partition`, + '$."double_array_data"') as double_array_data, + json_extract(`partition`, + '$."DATA"') as `DATA`, + json_extract(`partition`, + '$."column___with__quotes"') as `column__'with"_quotes`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`nested_stream_with_co__lting_into_long_names` as table_alias +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and `partition` is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql new file mode 100644 index 000000000000..f5b912cd6800 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_nested_strea__nto_long_names_hashid, + double_array_data, + `DATA`, + `column__'with"_quotes`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab1` +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql new file mode 100644 index 000000000000..bf2e57812e3d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_nested_strea__nto_long_names_hashid as char), ''), '-', coalesce(cast(double_array_data as char), ''), '-', coalesce(cast(`DATA` as char), ''), '-', coalesce(cast(`column__'with"_quotes` as char), '')) as char)) as _airbyte_partition_hashid, + tmp.* +from _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab2` tmp +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql new file mode 100644 index 000000000000..5dd3b6c8ce6f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql @@ -0,0 +1,64 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * pow(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 2 + order by generated_number + + + ), + joined as ( + select + _airbyte_partition_hashid as _airbyte_hashid, + + json_extract(double_array_data, concat("$[", numbers.generated_number - 1, "][0]")) as _airbyte_nested_data + from test_normalization.`nested_stream_with_co___long_names_partition` + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in nested_stream_with_co___long_names_partition.double_array_data + where numbers.generated_number <= json_length(double_array_data) + ) +select + _airbyte_partition_hashid, + json_value(_airbyte_nested_data, + '$."id"') as id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`nested_stream_with_co___long_names_partition` as table_alias +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and double_array_data is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql new file mode 100644 index 000000000000..3280a736e4bc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql @@ -0,0 +1,16 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(id as char) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab1` +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql new file mode 100644 index 000000000000..f14064aa73ee --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as char), ''), '-', coalesce(cast(id as char), '')) as char)) as _airbyte_double_array_data_hashid, + tmp.* +from _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab2` tmp +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql new file mode 100644 index 000000000000..19ae65abddfd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql @@ -0,0 +1,64 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * pow(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 1 + order by generated_number + + + ), + joined as ( + select + _airbyte_partition_hashid as _airbyte_hashid, + + json_extract(`DATA`, concat("$[", numbers.generated_number - 1, "][0]")) as _airbyte_nested_data + from test_normalization.`nested_stream_with_co___long_names_partition` + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in nested_stream_with_co___long_names_partition.`DATA` + where numbers.generated_number <= json_length(`DATA`) + ) +select + _airbyte_partition_hashid, + json_value(_airbyte_nested_data, + '$."currency"') as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`nested_stream_with_co___long_names_partition` as table_alias +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and `DATA` is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql new file mode 100644 index 000000000000..2a28642a67f8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql @@ -0,0 +1,16 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as char) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab1` +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql new file mode 100644 index 000000000000..14553972ba13 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as char), ''), '-', coalesce(cast(currency as char), '')) as char)) as _airbyte_data_hashid, + tmp.* +from _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab2` tmp +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql new file mode 100644 index 000000000000..c7861eb82bd9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql @@ -0,0 +1,64 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * pow(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 1 + order by generated_number + + + ), + joined as ( + select + _airbyte_partition_hashid as _airbyte_hashid, + + json_extract(`column__'with"_quotes`, concat("$[", numbers.generated_number - 1, "][0]")) as _airbyte_nested_data + from test_normalization.`nested_stream_with_co___long_names_partition` + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in nested_stream_with_co___long_names_partition.`column__'with"_quotes` + where numbers.generated_number <= json_length(`column__'with"_quotes`) + ) +select + _airbyte_partition_hashid, + json_value(_airbyte_nested_data, + '$."currency"') as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`nested_stream_with_co___long_names_partition` as table_alias +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and `column__'with"_quotes` is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql new file mode 100644 index 000000000000..6c71fa4683f5 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql @@ -0,0 +1,16 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as char) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab1` +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql new file mode 100644 index 000000000000..b8a160636ce2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as char), ''), '-', coalesce(cast(currency as char), '')) as char)) as _airbyte_column___with__quotes_hashid, + tmp.* +from _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab2` tmp +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql new file mode 100644 index 000000000000..c428b52dd85c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."date"') as `date`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_non_nest__lting_into_long_names as table_alias +-- non_nested_stream_wit__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql new file mode 100644 index 000000000000..0957dfcb71cb --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql @@ -0,0 +1,16 @@ + + create view _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as char) as id, + cast(`date` as char) as `date`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab1` +-- non_nested_stream_wit__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql new file mode 100644 index 000000000000..9122165fc08b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`date` as char), '')) as char)) as _airbyte_non_nested_s__nto_long_names_hashid, + tmp.* +from _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab2` tmp +-- non_nested_stream_wit__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql new file mode 100644 index 000000000000..46c85dc6a760 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql @@ -0,0 +1,19 @@ + + create view _airbyte_test_normalization.`unnest_alias_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_extract(_airbyte_data, + '$."children"') as children, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_unnest_alias as table_alias +-- unnest_alias +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql new file mode 100644 index 000000000000..d0321601fc92 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql @@ -0,0 +1,19 @@ + + create view _airbyte_test_normalization.`unnest_alias_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + signed +) as id, + children, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`unnest_alias_ab1` +-- unnest_alias +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql new file mode 100644 index 000000000000..9819d4b985d1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql @@ -0,0 +1,12 @@ + + create view _airbyte_test_normalization.`unnest_alias_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(children as char), '')) as char)) as _airbyte_unnest_alias_hashid, + tmp.* +from _airbyte_test_normalization.`unnest_alias_ab2` tmp +-- unnest_alias +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql new file mode 100644 index 000000000000..d781016d8076 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql @@ -0,0 +1,68 @@ + + create view _airbyte_test_normalization.`unnest_alias_children_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * pow(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 2 + order by generated_number + + + ), + joined as ( + select + _airbyte_unnest_alias_hashid as _airbyte_hashid, + + json_extract(children, concat("$[", numbers.generated_number - 1, "][0]")) as _airbyte_nested_data + from test_normalization.`unnest_alias` + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in unnest_alias.children + where numbers.generated_number <= json_length(children) + ) +select + _airbyte_unnest_alias_hashid, + json_value(_airbyte_nested_data, + '$."ab_id"') as ab_id, + + json_extract(_airbyte_nested_data, + '$."owner"') + as `owner`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`unnest_alias` as table_alias +-- children at unnest_alias/children +left join joined on _airbyte_unnest_alias_hashid = joined._airbyte_hashid +where 1 = 1 +and children is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql new file mode 100644 index 000000000000..a2e785ac473e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql @@ -0,0 +1,19 @@ + + create view _airbyte_test_normalization.`unnest_alias_children_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_unnest_alias_hashid, + cast(ab_id as + signed +) as ab_id, + cast(`owner` as json) as `owner`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`unnest_alias_children_ab1` +-- children at unnest_alias/children +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql new file mode 100644 index 000000000000..a10d4db5544e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`unnest_alias_children_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_unnest_alias_hashid as char), ''), '-', coalesce(cast(ab_id as char), ''), '-', coalesce(cast(`owner` as char), '')) as char)) as _airbyte_children_hashid, + tmp.* +from _airbyte_test_normalization.`unnest_alias_children_ab2` tmp +-- children at unnest_alias/children +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql new file mode 100644 index 000000000000..942b77e12e1d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`unnest_alias_children_owner_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_children_hashid, + json_value(`owner`, + '$."owner_id"') as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`unnest_alias_children` as table_alias +-- owner at unnest_alias/children/owner +where 1 = 1 +and `owner` is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql new file mode 100644 index 000000000000..1f95376e9d66 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`unnest_alias_children_owner_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_children_hashid, + cast(owner_id as + signed +) as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`unnest_alias_children_owner_ab1` +-- owner at unnest_alias/children/owner +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql new file mode 100644 index 000000000000..923718fab124 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`unnest_alias_children_owner_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_children_hashid as char), ''), '-', coalesce(cast(owner_id as char), '')) as char)) as _airbyte_owner_hashid, + tmp.* +from _airbyte_test_normalization.`unnest_alias_children_owner_ab2` tmp +-- owner at unnest_alias/children/owner +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql new file mode 100644 index 000000000000..72c98d266114 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."date"') as `date`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization_namespace._airbyte_raw_simple_s__lting_into_long_names as table_alias +-- simple_stream_with_na__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql new file mode 100644 index 000000000000..3869d2131aac --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql @@ -0,0 +1,16 @@ + + create view _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as char) as id, + cast(`date` as char) as `date`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab1` +-- simple_stream_with_na__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql new file mode 100644 index 000000000000..aa7c1c724947 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`date` as char), '')) as char)) as _airbyte_simple_strea__nto_long_names_hashid, + tmp.* +from _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab2` tmp +-- simple_stream_with_na__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql new file mode 100644 index 000000000000..bfb90b5ac404 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql @@ -0,0 +1,70 @@ + + + create table + test_normalization.`nested_stream_with_co_1g_into_long_names_scd__dbt_tmp` + as ( + +with + +input_data as ( + select * + from _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab3` + -- nested_stream_with_co__lting_into_long_names from test_normalization._airbyte_raw_nested_s__lting_into_long_names +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_unique_key, + id, + `date`, + `partition`, + `date` as _airbyte_start_at, + lag(`date`) over ( + partition by id + order by + `date` is null asc, + `date` desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(`date`) over ( + partition by id + order by + `date` is null asc, + `date` desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_nested_strea__nto_long_names_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), '')) as char)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + `date`, + `partition`, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_nested_strea__nto_long_names_hashid +from dedup_data where _airbyte_row_num = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql new file mode 100644 index 000000000000..358de44385e3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql @@ -0,0 +1,24 @@ + + + create table + test_normalization.`nested_stream_with_co__lting_into_long_names__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + `date`, + `partition`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_nested_strea__nto_long_names_hashid +from test_normalization.`nested_stream_with_co_1g_into_long_names_scd` +-- nested_stream_with_co__lting_into_long_names from test_normalization._airbyte_raw_nested_s__lting_into_long_names +where 1 = 1 +and _airbyte_active_row = 1 + + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..cc143bf47b05 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql @@ -0,0 +1,21 @@ + + + create table + test_normalization.`unnest_alias__dbt_tmp` + as ( + +-- Final base SQL model +select + id, + children, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_unnest_alias_hashid +from _airbyte_test_normalization.`unnest_alias_ab3` +-- unnest_alias from test_normalization._airbyte_raw_unnest_alias +where 1 = 1 + + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql new file mode 100644 index 000000000000..717bc072a956 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`conflict_stream_array__dbt_tmp` + as ( + +-- Final base SQL model +select + id, + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_conflict_stream_array_hashid +from _airbyte_test_normalization.`conflict_stream_array_ab3` +-- conflict_stream_array from test_normalization._airbyte_raw_conflict_stream_array +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql new file mode 100644 index 000000000000..bdfe747bf5e7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`conflict_stream_name__dbt_tmp` + as ( + +-- Final base SQL model +select + id, + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_conflict_stream_name_hashid +from _airbyte_test_normalization.`conflict_stream_name_ab3` +-- conflict_stream_name from test_normalization._airbyte_raw_conflict_stream_name +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql new file mode 100644 index 000000000000..ded6eda50709 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`conflict_stream_name____conflict_stream_name__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_conflict_stream_name_2_hashid, + `groups`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_conflict_stream_name_3_hashid +from _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab3` +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from test_normalization.`conflict_stream_name_conflict_stream_name` +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql new file mode 100644 index 000000000000..a64da6fc2bba --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`conflict_stream_name_conflict_stream_name__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_conflict_stream_name_hashid, + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_conflict_stream_name_2_hashid +from _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab3` +-- conflict_stream_name at conflict_stream_name/conflict_stream_name from test_normalization.`conflict_stream_name` +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql new file mode 100644 index 000000000000..b79e0e4c08a4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`conflict_stream_scalar__dbt_tmp` + as ( + +-- Final base SQL model +select + id, + conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_conflict_stream_scalar_hashid +from _airbyte_test_normalization.`conflict_stream_scalar_ab3` +-- conflict_stream_scalar from test_normalization._airbyte_raw_conflict_stream_scalar +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql new file mode 100644 index 000000000000..2c30ff5645d3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql @@ -0,0 +1,22 @@ + + + create table + test_normalization.`nested_stream_with_co___long_names_partition__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_nested_strea__nto_long_names_hashid, + double_array_data, + `DATA`, + `column__'with"_quotes`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_partition_hashid +from _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab3` +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from test_normalization.`nested_stream_with_co__lting_into_long_names` +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql new file mode 100644 index 000000000000..cc91b90e22c9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`nested_stream_with_co___names_partition_data__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_data_hashid +from _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab3` +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from test_normalization.`nested_stream_with_co___long_names_partition` +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql new file mode 100644 index 000000000000..600adbbcf8b8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`nested_stream_with_co__column___with__quotes__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_column___with__quotes_hashid +from _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab3` +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from test_normalization.`nested_stream_with_co___long_names_partition` +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql new file mode 100644 index 000000000000..a584e5957c67 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`nested_stream_with_co__ion_double_array_data__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_partition_hashid, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_double_array_data_hashid +from _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab3` +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from test_normalization.`nested_stream_with_co___long_names_partition` +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql new file mode 100644 index 000000000000..9b1d5c0c792a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`non_nested_stream_wit__lting_into_long_names__dbt_tmp` + as ( + +-- Final base SQL model +select + id, + `date`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_non_nested_s__nto_long_names_hashid +from _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab3` +-- non_nested_stream_wit__lting_into_long_names from test_normalization._airbyte_raw_non_nest__lting_into_long_names +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql new file mode 100644 index 000000000000..62be462c50f8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -0,0 +1,21 @@ + + + create table + test_normalization.`unnest_alias_children__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_unnest_alias_hashid, + ab_id, + `owner`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_children_hashid +from _airbyte_test_normalization.`unnest_alias_children_ab3` +-- children at unnest_alias/children from test_normalization.`unnest_alias` +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql new file mode 100644 index 000000000000..5747da2ce362 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`unnest_alias_children_owner__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_children_hashid, + owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_owner_hashid +from _airbyte_test_normalization.`unnest_alias_children_owner_ab3` +-- owner at unnest_alias/children/owner from test_normalization.`unnest_alias_children` +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql new file mode 100644 index 000000000000..8a5111492bbf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization_namespace.`simple_stream_with_na__lting_into_long_names__dbt_tmp` + as ( + +-- Final base SQL model +select + id, + `date`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_simple_strea__nto_long_names_hashid +from _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab3` +-- simple_stream_with_na__lting_into_long_names from test_normalization_namespace._airbyte_raw_simple_s__lting_into_long_names +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql new file mode 100644 index 000000000000..f8c6c82e2fbc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql @@ -0,0 +1,12 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(`partition` as char), '')) as char)) as _airbyte_nested_strea__nto_long_names_hashid, + tmp.* +from _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab2` tmp +-- nested_stream_with_co__lting_into_long_names +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql deleted file mode 100644 index 14e4ec11e013..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql +++ /dev/null @@ -1,19 +0,0 @@ -with table_row_counts as ( - select distinct count(*) as row_count, 2 as expected_count - from {{ source('test_normalization', '_airbyte_raw_nested_s__lting_into_long_names') }} -union all - select distinct count(*) as row_count, 2 as expected_count - from {{ ref('nested_stream_with_co__lting_into_long_names') }} -union all - select distinct count(*) as row_count, 2 as expected_count - from {{ ref('nested_stream_with_co___long_names_partition') }} -union all - select count(distinct currency) as row_count, 1 as expected_count - from {{ ref('nested_stream_with_co___names_partition_data') }} --- union all --- select count(distinct id) as row_count, 3 as expected_count --- from {{ ref('nested_stream_with_co__ion_double_array_data') }} -) -select * -from table_row_counts -where row_count != expected_count diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql index d3df56ab3d35..a256ac12c37a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_array('_airbyte_data', ['conflict_stream_array'], ['conflict_stream_array']) }} as conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} as table_alias -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql index 2f0da1bb06e7..e770c48bd904 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_array_ab1') }} -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql index 0a92c655f92a..4c6cb6138a52 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('conflict_stream_array_ab2') }} tmp -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql index d6e2edb75ced..97ea3e982cbe 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql @@ -1,10 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_conflict_stream_name_hashid, {{ json_extract('table_alias', 'conflict_stream_name', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name') }} as table_alias -where conflict_stream_name is not null -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql index d31ae94a8814..88e3e7b5974a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_conflict_stream_name_hashid, cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name__2flict_stream_name_ab1') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql index dddba298b946..b17ecef4ec82 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('conflict_stream_name__2flict_stream_name_ab2') }} tmp -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql index e29eff6b3c01..851bc06e78ca 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql @@ -1,10 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_conflict_stream_name_2_hashid, {{ json_extract_scalar('conflict_stream_name', ['groups'], ['groups']) }} as {{ adapter.quote('groups') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_conflict_stream_name') }} as table_alias -where conflict_stream_name is not null -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql index e131f0fc01c1..3bb11ee28a2e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_conflict_stream_name_2_hashid, cast({{ adapter.quote('groups') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('groups') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name__3flict_stream_name_ab1') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql index 8d66a3a4b364..04a82c2387b4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('conflict_stream_name__3flict_stream_name_ab2') }} tmp -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql index 8556d7f50afb..9128769dd58b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract('table_alias', '_airbyte_data', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} as table_alias -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql index a1d77f379bce..7612525c0943 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_ab1') }} -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql index dc562af89ba1..b9d505d0445a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('conflict_stream_name_ab2') }} tmp -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql index be57970e399e..6169df7dc10b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['conflict_stream_scalar'], ['conflict_stream_scalar']) }} as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} as table_alias -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql index cf0d9deb51cd..51d157d48e14 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast(conflict_stream_scalar as {{ dbt_utils.type_bigint() }}) as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_scalar_ab1') }} -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql index 781c4bbfc391..f71e7c572052 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('conflict_stream_scalar_ab2') }} tmp -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql index 8f7cabd6d907..bcca84084504 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql @@ -1,10 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, {{ json_extract('table_alias', '_airbyte_data', ['partition'], ['partition']) }} as {{ adapter.quote('partition') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_nested_s__lting_into_long_names') }} as table_alias -- nested_stream_with_co__lting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql index 6b268cccc087..ec9afbc12ca6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql @@ -1,10 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast({{ adapter.quote('date') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('date') }}, cast({{ adapter.quote('partition') }} as {{ type_json() }}) as {{ adapter.quote('partition') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co_1g_into_long_names_ab1') }} -- nested_stream_with_co__lting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql deleted file mode 100644 index ec401f322147..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql +++ /dev/null @@ -1,12 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - adapter.quote('date'), - adapter.quote('partition'), - ]) }} as _airbyte_nested_strea__nto_long_names_hashid, - tmp.* -from {{ ref('nested_stream_with_co_1g_into_long_names_ab2') }} tmp --- nested_stream_with_co__lting_into_long_names - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab4.sql deleted file mode 100644 index 6d2cfb296ed7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_nested_strea__nto_long_names_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('nested_stream_with_co_1g_into_long_names_ab3') }} tmp --- nested_stream_with_co__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_s__lting_into_long_names') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql index 31f6b6a06672..686acef42df9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql @@ -1,12 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_nested_strea__nto_long_names_hashid, {{ json_extract_array(adapter.quote('partition'), ['double_array_data'], ['double_array_data']) }} as double_array_data, {{ json_extract_array(adapter.quote('partition'), ['DATA'], ['DATA']) }} as {{ adapter.quote('DATA') }}, {{ json_extract_array(adapter.quote('partition'), ['column`_\'with"_quotes'], ['column___with__quotes']) }} as {{ adapter.quote('column__\'with"_quotes') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co__lting_into_long_names') }} as table_alias -where {{ adapter.quote('partition') }} is not null -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and {{ adapter.quote('partition') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql index 81aae267055d..17847b387ff8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql @@ -1,11 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_nested_strea__nto_long_names_hashid, double_array_data, {{ adapter.quote('DATA') }}, {{ adapter.quote('column__\'with"_quotes') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co_2g_names_partition_ab1') }} -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql index 579a124df198..cd19c3cd4d2a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -10,4 +14,5 @@ select tmp.* from {{ ref('nested_stream_with_co_2g_names_partition_ab2') }} tmp -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql index 5b3a0601175b..a191b72022c5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql @@ -1,12 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_co___long_names_partition', 'partition', 'double_array_data') }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value('double_array_data'), ['id'], ['id']) }} as id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co___long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', 'double_array_data') }} -where double_array_data is not null -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +{{ cross_join_unnest('partition', 'double_array_data') }} +where 1 = 1 +and double_array_data is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql index b4357e9d1f1e..685ebd8cf4f0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(id as {{ dbt_utils.type_string() }}) as id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co_3double_array_data_ab1') }} -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql index a8aa152099ce..d3d654a0b267 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('nested_stream_with_co_3double_array_data_ab2') }} tmp -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql index d5c4708f7e07..46f5b7c11ae2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql @@ -1,12 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_co___long_names_partition', 'partition', adapter.quote('DATA')) }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value(adapter.quote('DATA')), ['currency'], ['currency']) }} as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co___long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', adapter.quote('DATA')) }} -where {{ adapter.quote('DATA') }} is not null -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +{{ cross_join_unnest('partition', adapter.quote('DATA')) }} +where 1 = 1 +and {{ adapter.quote('DATA') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql index baad4bbf0f14..b6912038bca1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(currency as {{ dbt_utils.type_string() }}) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co_3es_partition_data_ab1') }} -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql index ff7df8bea771..cc049c42b2c4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('nested_stream_with_co_3es_partition_data_ab2') }} tmp -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql index f8b1f55fc4df..8b23b31f54b9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql @@ -1,12 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_co___long_names_partition', 'partition', adapter.quote('column__\'with"_quotes')) }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value(adapter.quote('column__\'with"_quotes')), ['currency'], ['currency']) }} as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co___long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', adapter.quote('column__\'with"_quotes')) }} -where {{ adapter.quote('column__\'with"_quotes') }} is not null -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +{{ cross_join_unnest('partition', adapter.quote('column__\'with"_quotes')) }} +where 1 = 1 +and {{ adapter.quote('column__\'with"_quotes') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql index 32acb349e636..4596c0998414 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(currency as {{ dbt_utils.type_string() }}) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co_3mn___with__quotes_ab1') }} -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql index fed352d1f303..640685bc8c6a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('nested_stream_with_co_3mn___with__quotes_ab2') }} tmp -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql index 6575f5003ddf..4e89cc3ec03c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_non_nest__lting_into_long_names') }} as table_alias -- non_nested_stream_wit__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql index d8d262e97685..c1bc0cb6750f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast({{ adapter.quote('date') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('date') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('non_nested_stream_wit_1g_into_long_names_ab1') }} -- non_nested_stream_wit__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql index 98a4ac30d1be..81d16c7714e9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('non_nested_stream_wit_1g_into_long_names_ab2') }} tmp -- non_nested_stream_wit__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql index 15b34c1ca7d4..a7877e1d2dde 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_array('_airbyte_data', ['children'], ['children']) }} as children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} as table_alias -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql index a416538b5422..b397e46d7b04 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_ab1') }} -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql index 0c6703933b10..9f2bd6c9d3af 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,6 @@ select tmp.* from {{ ref('unnest_alias_ab2') }} tmp -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql index 2237b274e29b..0ae376bf5c5e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql @@ -1,13 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('unnest_alias', 'unnest_alias', 'children') }} select _airbyte_unnest_alias_hashid, {{ json_extract_scalar(unnested_column_value('children'), ['ab_id'], ['ab_id']) }} as ab_id, {{ json_extract('', unnested_column_value('children'), ['owner'], ['owner']) }} as {{ adapter.quote('owner') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias') }} as table_alias -{{ cross_join_unnest('unnest_alias', 'children') }} -where children is not null -- children at unnest_alias/children +{{ cross_join_unnest('unnest_alias', 'children') }} +where 1 = 1 +and children is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql index 435447589db2..4e285cc11ccc 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql @@ -1,10 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_unnest_alias_hashid, cast(ab_id as {{ dbt_utils.type_bigint() }}) as ab_id, cast({{ adapter.quote('owner') }} as {{ type_json() }}) as {{ adapter.quote('owner') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children_ab1') }} -- children at unnest_alias/children +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql index e1fa4b470cc8..71b013d95e95 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -9,4 +13,5 @@ select tmp.* from {{ ref('unnest_alias_children_ab2') }} tmp -- children at unnest_alias/children +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql index 9085180f4824..833b7054705a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql @@ -1,10 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_children_hashid, {{ json_extract_scalar(adapter.quote('owner'), ['owner_id'], ['owner_id']) }} as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children') }} as table_alias -where {{ adapter.quote('owner') }} is not null -- owner at unnest_alias/children/owner +where 1 = 1 +and {{ adapter.quote('owner') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql index a0f474c62db7..109a7652533a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_children_hashid, cast(owner_id as {{ dbt_utils.type_bigint() }}) as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children_owner_ab1') }} -- owner at unnest_alias/children/owner +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql index c5f6562c34c2..53a716a401c1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('unnest_alias_children_owner_ab2') }} tmp -- owner at unnest_alias/children/owner +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql index ab36933bcef0..15a7e0bc2774 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization_namespace', '_airbyte_raw_simple_s__lting_into_long_names') }} as table_alias -- simple_stream_with_na__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql index 0cd875e93da8..b37432da5333 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast({{ adapter.quote('date') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('date') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('simple_stream_with_na_1g_into_long_names_ab1') }} -- simple_stream_with_na__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql index 281544a38863..bfe41baca551 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('simple_stream_with_na_1g_into_long_names_ab2') }} tmp -- simple_stream_with_na__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql new file mode 100644 index 000000000000..98783c3a173d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql @@ -0,0 +1,107 @@ +{{ config( + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('nested_stream_with_co_1g_into_long_names_ab3') }} + -- nested_stream_with_co__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_s__lting_into_long_names') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('nested_stream_with_co_1g_into_long_names_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('nested_stream_with_co_1g_into_long_names_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('nested_stream_with_co_1g_into_long_names_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('nested_stream_with_co_1g_into_long_names_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('nested_stream_with_co_1g_into_long_names_ab3') }} + -- nested_stream_with_co__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_s__lting_into_long_names') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + {{ adapter.quote('date') }}, + {{ adapter.quote('partition') }}, + {{ adapter.quote('date') }} as _airbyte_start_at, + lag({{ adapter.quote('date') }}) over ( + partition by id + order by + {{ adapter.quote('date') }} is null asc, + {{ adapter.quote('date') }} desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag({{ adapter.quote('date') }}) over ( + partition by id + order by + {{ adapter.quote('date') }} is null asc, + {{ adapter.quote('date') }} desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_nested_strea__nto_long_names_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + {{ adapter.quote('date') }}, + {{ adapter.quote('partition') }}, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_nested_strea__nto_long_names_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql new file mode 100644 index 000000000000..bb554d43a98d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql @@ -0,0 +1,21 @@ +{{ config( + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + id, + {{ adapter.quote('date') }}, + {{ adapter.quote('partition') }}, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_nested_strea__nto_long_names_hashid +from {{ ref('nested_stream_with_co_1g_into_long_names_scd') }} +-- nested_stream_with_co__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_s__lting_into_long_names') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..dad253380255 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql @@ -0,0 +1,18 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + id, + children, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_unnest_alias_hashid +from {{ ref('unnest_alias_ab3') }} +-- unnest_alias from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql index 9ffda6086e35..6e616cb068a4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, conflict_stream_array, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_array_hashid from {{ ref('conflict_stream_array_ab3') }} -- conflict_stream_array from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql index 8a83ccc01e31..6b70e1601f56 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_hashid from {{ ref('conflict_stream_name_ab3') }} -- conflict_stream_name from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql index ac40e0861b86..81f173664209 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_conflict_stream_name_2_hashid, {{ adapter.quote('groups') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_3_hashid from {{ ref('conflict_stream_name__3flict_stream_name_ab3') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name_conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql index ea20646f8121..be1b4fe446a4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_conflict_stream_name_hashid, conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_2_hashid from {{ ref('conflict_stream_name__2flict_stream_name_ab3') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql index 9d7acc846fd0..3e196df09e88 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, conflict_stream_scalar, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_scalar_hashid from {{ ref('conflict_stream_scalar_ab3') }} -- conflict_stream_scalar from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql deleted file mode 100644 index 71213c7a23fa..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - {{ adapter.quote('date') }}, - {{ adapter.quote('partition') }}, - {{ adapter.quote('date') }} as _airbyte_start_at, - lag({{ adapter.quote('date') }}) over ( - partition by id - order by {{ adapter.quote('date') }} is null asc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag({{ adapter.quote('date') }}) over ( - partition by id - order by {{ adapter.quote('date') }} is null asc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_nested_strea__nto_long_names_hashid -from {{ ref('nested_stream_with_co_1g_into_long_names_ab4') }} --- nested_stream_with_co__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_s__lting_into_long_names') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql index b64a0b856c0e..f5f0f16333f8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql @@ -1,12 +1,19 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_nested_strea__nto_long_names_hashid, double_array_data, {{ adapter.quote('DATA') }}, {{ adapter.quote('column__\'with"_quotes') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_partition_hashid from {{ ref('nested_stream_with_co_2g_names_partition_ab3') }} -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from {{ ref('nested_stream_with_co__lting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql index eec219cee615..5219d4bf1eb9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_data_hashid from {{ ref('nested_stream_with_co_3es_partition_data_ab3') }} -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from {{ ref('nested_stream_with_co___long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql index b7c647643b54..b56e3f5b5da3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_column___with__quotes_hashid from {{ ref('nested_stream_with_co_3mn___with__quotes_ab3') }} -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from {{ ref('nested_stream_with_co___long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql index eb0f195e2786..1cd2763acbac 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, id, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_double_array_data_hashid from {{ ref('nested_stream_with_co_3double_array_data_ab3') }} -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from {{ ref('nested_stream_with_co___long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql deleted file mode 100644 index a35988d6c18f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql +++ /dev/null @@ -1,12 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - {{ adapter.quote('date') }}, - {{ adapter.quote('partition') }}, - _airbyte_emitted_at, - _airbyte_nested_strea__nto_long_names_hashid -from {{ ref('nested_stream_with_co_1g_into_long_names_scd') }} --- nested_stream_with_co__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_s__lting_into_long_names') }} -where _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql index 12fcd9b638fa..80f9bdb879a3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, {{ adapter.quote('date') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_non_nested_s__nto_long_names_hashid from {{ ref('non_nested_stream_wit_1g_into_long_names_ab3') }} -- non_nested_stream_wit__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_non_nest__lting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql deleted file mode 100644 index 7a570c3836cb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,10 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - children, - _airbyte_emitted_at, - _airbyte_unnest_alias_hashid -from {{ ref('unnest_alias_ab3') }} --- unnest_alias from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql index 9ce667c8b142..59c7f1b6c50d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -1,11 +1,18 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_unnest_alias_hashid, ab_id, {{ adapter.quote('owner') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_children_hashid from {{ ref('unnest_alias_children_ab3') }} -- children at unnest_alias/children from {{ ref('unnest_alias') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql index 0ccf8e33d8d6..e6710c9eb232 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_children_hashid, owner_id, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_owner_hashid from {{ ref('unnest_alias_children_owner_ab3') }} -- owner at unnest_alias/children/owner from {{ ref('unnest_alias_children') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql index fdd42e4ce199..da684535a739 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization_namespace", tags=["top-level"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization_namespace", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, {{ adapter.quote('date') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_simple_strea__nto_long_names_hashid from {{ ref('simple_stream_with_na_1g_into_long_names_ab3') }} -- simple_stream_with_na__lting_into_long_names from {{ source('test_normalization_namespace', '_airbyte_raw_simple_s__lting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql new file mode 100644 index 000000000000..1e93090d9c40 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql @@ -0,0 +1,18 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + adapter.quote('date'), + adapter.quote('partition'), + ]) }} as _airbyte_nested_strea__nto_long_names_hashid, + tmp.* +from {{ ref('nested_stream_with_co_1g_into_long_names_ab2') }} tmp +-- nested_stream_with_co__lting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql new file mode 100644 index 000000000000..b08ffaf4f715 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`conflict_stream_array_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_extract(_airbyte_data, + '$."conflict_stream_array"') as conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_conflict_stream_array as table_alias +-- conflict_stream_array +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql new file mode 100644 index 000000000000..44b15b327edd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql @@ -0,0 +1,16 @@ + + create view _airbyte_test_normalization.`conflict_stream_array_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as char) as id, + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`conflict_stream_array_ab1` +-- conflict_stream_array +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql new file mode 100644 index 000000000000..857eb62be3f6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`conflict_stream_array_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(conflict_stream_array as char), '')) as char)) as _airbyte_conflict_stream_array_hashid, + tmp.* +from _airbyte_test_normalization.`conflict_stream_array_ab2` tmp +-- conflict_stream_array +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql new file mode 100644 index 000000000000..e3edc7461b2d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql @@ -0,0 +1,20 @@ + + create view _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_hashid, + + json_extract(table_alias.conflict_stream_name, + '$."conflict_stream_name"') + as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`conflict_stream_name` as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql new file mode 100644 index 000000000000..16def64e5011 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql @@ -0,0 +1,16 @@ + + create view _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_hashid, + cast(conflict_stream_name as json) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab1` +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql new file mode 100644 index 000000000000..99164479818a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_conflict_stream_name_hashid as char), ''), '-', coalesce(cast(conflict_stream_name as char), '')) as char)) as _airbyte_conflict_stream_name_2_hashid, + tmp.* +from _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab2` tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql new file mode 100644 index 000000000000..94cef187f111 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_2_hashid, + json_value(conflict_stream_name, + '$."groups"') as `groups`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`conflict_stream_name_conflict_stream_name` as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql new file mode 100644 index 000000000000..222256b4cc4e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql @@ -0,0 +1,16 @@ + + create view _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_2_hashid, + cast(`groups` as char) as `groups`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab1` +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql new file mode 100644 index 000000000000..2442380dac27 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as char), ''), '-', coalesce(cast(`groups` as char), '')) as char)) as _airbyte_conflict_stream_name_3_hashid, + tmp.* +from _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab2` tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..f0cefd4a6acf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql @@ -0,0 +1,20 @@ + + create view _airbyte_test_normalization.`conflict_stream_name_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + + json_extract(table_alias._airbyte_data, + '$."conflict_stream_name"') + as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_conflict_stream_name as table_alias +-- conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..b4f1e6a94ea6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql @@ -0,0 +1,16 @@ + + create view _airbyte_test_normalization.`conflict_stream_name_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as char) as id, + cast(conflict_stream_name as json) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`conflict_stream_name_ab1` +-- conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..8a7e62bbfafc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`conflict_stream_name_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(conflict_stream_name as char), '')) as char)) as _airbyte_conflict_stream_name_hashid, + tmp.* +from _airbyte_test_normalization.`conflict_stream_name_ab2` tmp +-- conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql new file mode 100644 index 000000000000..a30f57d7d4ed --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`conflict_stream_scalar_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."conflict_stream_scalar"') as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_conflict_stream_scalar as table_alias +-- conflict_stream_scalar +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql new file mode 100644 index 000000000000..5822b0221381 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`conflict_stream_scalar_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as char) as id, + cast(conflict_stream_scalar as + signed +) as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`conflict_stream_scalar_ab1` +-- conflict_stream_scalar +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql new file mode 100644 index 000000000000..6b31dc6f9b85 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`conflict_stream_scalar_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(conflict_stream_scalar as char), '')) as char)) as _airbyte_conflict_stream_scalar_hashid, + tmp.* +from _airbyte_test_normalization.`conflict_stream_scalar_ab2` tmp +-- conflict_stream_scalar +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql new file mode 100644 index 000000000000..2a4990a4233d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql @@ -0,0 +1,23 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."date"') as `date`, + + json_extract(table_alias._airbyte_data, + '$."partition"') + as `partition`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_nested_s__lting_into_long_names as table_alias +-- nested_stream_with_co__lting_into_long_names +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql new file mode 100644 index 000000000000..a5cd56efead8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as char) as id, + cast(`date` as char) as `date`, + cast(`partition` as json) as `partition`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab1` +-- nested_stream_with_co__lting_into_long_names +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql new file mode 100644 index 000000000000..328e7097da14 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql @@ -0,0 +1,22 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_nested_strea__nto_long_names_hashid, + json_extract(`partition`, + '$."double_array_data"') as double_array_data, + json_extract(`partition`, + '$."DATA"') as `DATA`, + json_extract(`partition`, + '$."column___with__quotes"') as `column__'with"_quotes`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`nested_stream_with_co__lting_into_long_names` as table_alias +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and `partition` is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql new file mode 100644 index 000000000000..f5b912cd6800 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_nested_strea__nto_long_names_hashid, + double_array_data, + `DATA`, + `column__'with"_quotes`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab1` +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql new file mode 100644 index 000000000000..bf2e57812e3d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_nested_strea__nto_long_names_hashid as char), ''), '-', coalesce(cast(double_array_data as char), ''), '-', coalesce(cast(`DATA` as char), ''), '-', coalesce(cast(`column__'with"_quotes` as char), '')) as char)) as _airbyte_partition_hashid, + tmp.* +from _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab2` tmp +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql new file mode 100644 index 000000000000..5dd3b6c8ce6f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql @@ -0,0 +1,64 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * pow(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 2 + order by generated_number + + + ), + joined as ( + select + _airbyte_partition_hashid as _airbyte_hashid, + + json_extract(double_array_data, concat("$[", numbers.generated_number - 1, "][0]")) as _airbyte_nested_data + from test_normalization.`nested_stream_with_co___long_names_partition` + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in nested_stream_with_co___long_names_partition.double_array_data + where numbers.generated_number <= json_length(double_array_data) + ) +select + _airbyte_partition_hashid, + json_value(_airbyte_nested_data, + '$."id"') as id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`nested_stream_with_co___long_names_partition` as table_alias +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and double_array_data is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql new file mode 100644 index 000000000000..3280a736e4bc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql @@ -0,0 +1,16 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(id as char) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab1` +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql new file mode 100644 index 000000000000..f14064aa73ee --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as char), ''), '-', coalesce(cast(id as char), '')) as char)) as _airbyte_double_array_data_hashid, + tmp.* +from _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab2` tmp +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql new file mode 100644 index 000000000000..19ae65abddfd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql @@ -0,0 +1,64 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * pow(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 1 + order by generated_number + + + ), + joined as ( + select + _airbyte_partition_hashid as _airbyte_hashid, + + json_extract(`DATA`, concat("$[", numbers.generated_number - 1, "][0]")) as _airbyte_nested_data + from test_normalization.`nested_stream_with_co___long_names_partition` + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in nested_stream_with_co___long_names_partition.`DATA` + where numbers.generated_number <= json_length(`DATA`) + ) +select + _airbyte_partition_hashid, + json_value(_airbyte_nested_data, + '$."currency"') as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`nested_stream_with_co___long_names_partition` as table_alias +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and `DATA` is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql new file mode 100644 index 000000000000..2a28642a67f8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql @@ -0,0 +1,16 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as char) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab1` +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql new file mode 100644 index 000000000000..14553972ba13 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as char), ''), '-', coalesce(cast(currency as char), '')) as char)) as _airbyte_data_hashid, + tmp.* +from _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab2` tmp +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql new file mode 100644 index 000000000000..c7861eb82bd9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql @@ -0,0 +1,64 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * pow(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 1 + order by generated_number + + + ), + joined as ( + select + _airbyte_partition_hashid as _airbyte_hashid, + + json_extract(`column__'with"_quotes`, concat("$[", numbers.generated_number - 1, "][0]")) as _airbyte_nested_data + from test_normalization.`nested_stream_with_co___long_names_partition` + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in nested_stream_with_co___long_names_partition.`column__'with"_quotes` + where numbers.generated_number <= json_length(`column__'with"_quotes`) + ) +select + _airbyte_partition_hashid, + json_value(_airbyte_nested_data, + '$."currency"') as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`nested_stream_with_co___long_names_partition` as table_alias +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and `column__'with"_quotes` is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql new file mode 100644 index 000000000000..6c71fa4683f5 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql @@ -0,0 +1,16 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as char) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab1` +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql new file mode 100644 index 000000000000..b8a160636ce2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as char), ''), '-', coalesce(cast(currency as char), '')) as char)) as _airbyte_column___with__quotes_hashid, + tmp.* +from _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab2` tmp +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql new file mode 100644 index 000000000000..c428b52dd85c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."date"') as `date`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_non_nest__lting_into_long_names as table_alias +-- non_nested_stream_wit__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql new file mode 100644 index 000000000000..0957dfcb71cb --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql @@ -0,0 +1,16 @@ + + create view _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as char) as id, + cast(`date` as char) as `date`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab1` +-- non_nested_stream_wit__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql new file mode 100644 index 000000000000..9122165fc08b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`date` as char), '')) as char)) as _airbyte_non_nested_s__nto_long_names_hashid, + tmp.* +from _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab2` tmp +-- non_nested_stream_wit__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql new file mode 100644 index 000000000000..46c85dc6a760 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql @@ -0,0 +1,19 @@ + + create view _airbyte_test_normalization.`unnest_alias_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_extract(_airbyte_data, + '$."children"') as children, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_unnest_alias as table_alias +-- unnest_alias +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql new file mode 100644 index 000000000000..d0321601fc92 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql @@ -0,0 +1,19 @@ + + create view _airbyte_test_normalization.`unnest_alias_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + signed +) as id, + children, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`unnest_alias_ab1` +-- unnest_alias +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql new file mode 100644 index 000000000000..9819d4b985d1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql @@ -0,0 +1,12 @@ + + create view _airbyte_test_normalization.`unnest_alias_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(children as char), '')) as char)) as _airbyte_unnest_alias_hashid, + tmp.* +from _airbyte_test_normalization.`unnest_alias_ab2` tmp +-- unnest_alias +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql new file mode 100644 index 000000000000..d781016d8076 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql @@ -0,0 +1,68 @@ + + create view _airbyte_test_normalization.`unnest_alias_children_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * pow(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 2 + order by generated_number + + + ), + joined as ( + select + _airbyte_unnest_alias_hashid as _airbyte_hashid, + + json_extract(children, concat("$[", numbers.generated_number - 1, "][0]")) as _airbyte_nested_data + from test_normalization.`unnest_alias` + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in unnest_alias.children + where numbers.generated_number <= json_length(children) + ) +select + _airbyte_unnest_alias_hashid, + json_value(_airbyte_nested_data, + '$."ab_id"') as ab_id, + + json_extract(_airbyte_nested_data, + '$."owner"') + as `owner`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`unnest_alias` as table_alias +-- children at unnest_alias/children +left join joined on _airbyte_unnest_alias_hashid = joined._airbyte_hashid +where 1 = 1 +and children is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql new file mode 100644 index 000000000000..a2e785ac473e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql @@ -0,0 +1,19 @@ + + create view _airbyte_test_normalization.`unnest_alias_children_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_unnest_alias_hashid, + cast(ab_id as + signed +) as ab_id, + cast(`owner` as json) as `owner`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`unnest_alias_children_ab1` +-- children at unnest_alias/children +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql new file mode 100644 index 000000000000..a10d4db5544e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`unnest_alias_children_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_unnest_alias_hashid as char), ''), '-', coalesce(cast(ab_id as char), ''), '-', coalesce(cast(`owner` as char), '')) as char)) as _airbyte_children_hashid, + tmp.* +from _airbyte_test_normalization.`unnest_alias_children_ab2` tmp +-- children at unnest_alias/children +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql new file mode 100644 index 000000000000..942b77e12e1d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`unnest_alias_children_owner_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_children_hashid, + json_value(`owner`, + '$."owner_id"') as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`unnest_alias_children` as table_alias +-- owner at unnest_alias/children/owner +where 1 = 1 +and `owner` is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql new file mode 100644 index 000000000000..1f95376e9d66 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`unnest_alias_children_owner_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_children_hashid, + cast(owner_id as + signed +) as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`unnest_alias_children_owner_ab1` +-- owner at unnest_alias/children/owner +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql new file mode 100644 index 000000000000..923718fab124 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`unnest_alias_children_owner_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_children_hashid as char), ''), '-', coalesce(cast(owner_id as char), '')) as char)) as _airbyte_owner_hashid, + tmp.* +from _airbyte_test_normalization.`unnest_alias_children_owner_ab2` tmp +-- owner at unnest_alias/children/owner +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql new file mode 100644 index 000000000000..72c98d266114 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."date"') as `date`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization_namespace._airbyte_raw_simple_s__lting_into_long_names as table_alias +-- simple_stream_with_na__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql new file mode 100644 index 000000000000..3869d2131aac --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql @@ -0,0 +1,16 @@ + + create view _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as char) as id, + cast(`date` as char) as `date`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab1` +-- simple_stream_with_na__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql new file mode 100644 index 000000000000..aa7c1c724947 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`date` as char), '')) as char)) as _airbyte_simple_strea__nto_long_names_hashid, + tmp.* +from _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab2` tmp +-- simple_stream_with_na__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql new file mode 100644 index 000000000000..bfb90b5ac404 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql @@ -0,0 +1,70 @@ + + + create table + test_normalization.`nested_stream_with_co_1g_into_long_names_scd__dbt_tmp` + as ( + +with + +input_data as ( + select * + from _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab3` + -- nested_stream_with_co__lting_into_long_names from test_normalization._airbyte_raw_nested_s__lting_into_long_names +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_unique_key, + id, + `date`, + `partition`, + `date` as _airbyte_start_at, + lag(`date`) over ( + partition by id + order by + `date` is null asc, + `date` desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(`date`) over ( + partition by id + order by + `date` is null asc, + `date` desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_nested_strea__nto_long_names_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), '')) as char)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + `date`, + `partition`, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_nested_strea__nto_long_names_hashid +from dedup_data where _airbyte_row_num = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql new file mode 100644 index 000000000000..358de44385e3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql @@ -0,0 +1,24 @@ + + + create table + test_normalization.`nested_stream_with_co__lting_into_long_names__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + `date`, + `partition`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_nested_strea__nto_long_names_hashid +from test_normalization.`nested_stream_with_co_1g_into_long_names_scd` +-- nested_stream_with_co__lting_into_long_names from test_normalization._airbyte_raw_nested_s__lting_into_long_names +where 1 = 1 +and _airbyte_active_row = 1 + + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..cc143bf47b05 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql @@ -0,0 +1,21 @@ + + + create table + test_normalization.`unnest_alias__dbt_tmp` + as ( + +-- Final base SQL model +select + id, + children, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_unnest_alias_hashid +from _airbyte_test_normalization.`unnest_alias_ab3` +-- unnest_alias from test_normalization._airbyte_raw_unnest_alias +where 1 = 1 + + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql new file mode 100644 index 000000000000..717bc072a956 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`conflict_stream_array__dbt_tmp` + as ( + +-- Final base SQL model +select + id, + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_conflict_stream_array_hashid +from _airbyte_test_normalization.`conflict_stream_array_ab3` +-- conflict_stream_array from test_normalization._airbyte_raw_conflict_stream_array +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql new file mode 100644 index 000000000000..bdfe747bf5e7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`conflict_stream_name__dbt_tmp` + as ( + +-- Final base SQL model +select + id, + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_conflict_stream_name_hashid +from _airbyte_test_normalization.`conflict_stream_name_ab3` +-- conflict_stream_name from test_normalization._airbyte_raw_conflict_stream_name +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql new file mode 100644 index 000000000000..ded6eda50709 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`conflict_stream_name____conflict_stream_name__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_conflict_stream_name_2_hashid, + `groups`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_conflict_stream_name_3_hashid +from _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab3` +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from test_normalization.`conflict_stream_name_conflict_stream_name` +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql new file mode 100644 index 000000000000..a64da6fc2bba --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`conflict_stream_name_conflict_stream_name__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_conflict_stream_name_hashid, + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_conflict_stream_name_2_hashid +from _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab3` +-- conflict_stream_name at conflict_stream_name/conflict_stream_name from test_normalization.`conflict_stream_name` +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql new file mode 100644 index 000000000000..b79e0e4c08a4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`conflict_stream_scalar__dbt_tmp` + as ( + +-- Final base SQL model +select + id, + conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_conflict_stream_scalar_hashid +from _airbyte_test_normalization.`conflict_stream_scalar_ab3` +-- conflict_stream_scalar from test_normalization._airbyte_raw_conflict_stream_scalar +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql new file mode 100644 index 000000000000..2c30ff5645d3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql @@ -0,0 +1,22 @@ + + + create table + test_normalization.`nested_stream_with_co___long_names_partition__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_nested_strea__nto_long_names_hashid, + double_array_data, + `DATA`, + `column__'with"_quotes`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_partition_hashid +from _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab3` +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from test_normalization.`nested_stream_with_co__lting_into_long_names` +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql new file mode 100644 index 000000000000..cc91b90e22c9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`nested_stream_with_co___names_partition_data__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_data_hashid +from _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab3` +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from test_normalization.`nested_stream_with_co___long_names_partition` +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql new file mode 100644 index 000000000000..600adbbcf8b8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`nested_stream_with_co__column___with__quotes__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_column___with__quotes_hashid +from _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab3` +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from test_normalization.`nested_stream_with_co___long_names_partition` +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql new file mode 100644 index 000000000000..a584e5957c67 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`nested_stream_with_co__ion_double_array_data__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_partition_hashid, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_double_array_data_hashid +from _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab3` +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from test_normalization.`nested_stream_with_co___long_names_partition` +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql new file mode 100644 index 000000000000..9b1d5c0c792a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`non_nested_stream_wit__lting_into_long_names__dbt_tmp` + as ( + +-- Final base SQL model +select + id, + `date`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_non_nested_s__nto_long_names_hashid +from _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab3` +-- non_nested_stream_wit__lting_into_long_names from test_normalization._airbyte_raw_non_nest__lting_into_long_names +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql new file mode 100644 index 000000000000..62be462c50f8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -0,0 +1,21 @@ + + + create table + test_normalization.`unnest_alias_children__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_unnest_alias_hashid, + ab_id, + `owner`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_children_hashid +from _airbyte_test_normalization.`unnest_alias_children_ab3` +-- children at unnest_alias/children from test_normalization.`unnest_alias` +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql new file mode 100644 index 000000000000..5747da2ce362 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`unnest_alias_children_owner__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_children_hashid, + owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_owner_hashid +from _airbyte_test_normalization.`unnest_alias_children_owner_ab3` +-- owner at unnest_alias/children/owner from test_normalization.`unnest_alias_children` +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql new file mode 100644 index 000000000000..8a5111492bbf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization_namespace.`simple_stream_with_na__lting_into_long_names__dbt_tmp` + as ( + +-- Final base SQL model +select + id, + `date`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_simple_strea__nto_long_names_hashid +from _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab3` +-- simple_stream_with_na__lting_into_long_names from test_normalization_namespace._airbyte_raw_simple_s__lting_into_long_names +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql new file mode 100644 index 000000000000..f8c6c82e2fbc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql @@ -0,0 +1,12 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(`partition` as char), '')) as char)) as _airbyte_nested_strea__nto_long_names_hashid, + tmp.* +from _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab2` tmp +-- nested_stream_with_co__lting_into_long_names +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql deleted file mode 100644 index 6fd2103d5692..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,21 +0,0 @@ - - create view _airbyte_test_normalization.`dedup_cdc_excluded_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_value(_airbyte_data, - '$."name"') as `name`, - json_value(_airbyte_data, - '$."column___with__quotes"') as `column__'with"_quotes`, - json_value(_airbyte_data, - '$."_ab_cdc_lsn"') as _ab_cdc_lsn, - json_value(_airbyte_data, - '$."_ab_cdc_updated_at"') as _ab_cdc_updated_at, - json_value(_airbyte_data, - '$."_ab_cdc_deleted_at"') as _ab_cdc_deleted_at, - _airbyte_emitted_at -from test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias --- dedup_cdc_excluded - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql deleted file mode 100644 index 394e3a59e8d7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,23 +0,0 @@ - - create view _airbyte_test_normalization.`dedup_cdc_excluded_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - signed -) as id, - cast(`name` as char) as `name`, - cast(`column__'with"_quotes` as char) as `column__'with"_quotes`, - cast(_ab_cdc_lsn as - float -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float -) as _ab_cdc_deleted_at, - _airbyte_emitted_at -from _airbyte_test_normalization.`dedup_cdc_excluded_ab1` --- dedup_cdc_excluded - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index ec1da44389ca..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,10 +0,0 @@ - - create view _airbyte_test_normalization.`dedup_cdc_excluded_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`name` as char), ''), '-', coalesce(cast(`column__'with"_quotes` as char), ''), '-', coalesce(cast(_ab_cdc_lsn as char), ''), '-', coalesce(cast(_ab_cdc_updated_at as char), ''), '-', coalesce(cast(_ab_cdc_deleted_at as char), '')) as char)) as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from _airbyte_test_normalization.`dedup_cdc_excluded_ab2` tmp --- dedup_cdc_excluded - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql deleted file mode 100644 index e65c6a32e65a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view _airbyte_test_normalization.`dedup_cdc_excluded_ab4__dbt_tmp` as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_cdc_excluded_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from _airbyte_test_normalization.`dedup_cdc_excluded_ab3` tmp --- dedup_cdc_excluded from test_normalization._airbyte_raw_dedup_cdc_excluded - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql deleted file mode 100644 index dae5bd72e4ab..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ /dev/null @@ -1,25 +0,0 @@ - - create view _airbyte_test_normalization.`dedup_exchange_rate_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_value(_airbyte_data, - '$."currency"') as currency, - json_value(_airbyte_data, - '$."date"') as `date`, - json_value(_airbyte_data, - '$."timestamp_col"') as timestamp_col, - json_value(_airbyte_data, - '$."HKD@spéçiäl & characters"') as `HKD@spéçiäl & characters`, - json_value(_airbyte_data, - '$."HKD_special___characters"') as hkd_special___characters, - json_value(_airbyte_data, - '$."NZD"') as nzd, - json_value(_airbyte_data, - '$."USD"') as usd, - _airbyte_emitted_at -from test_normalization._airbyte_raw_dedup_exchange_rate as table_alias --- dedup_exchange_rate - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql deleted file mode 100644 index a935dd0e7021..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ /dev/null @@ -1,28 +0,0 @@ - - create view _airbyte_test_normalization.`dedup_exchange_rate_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - signed -) as id, - cast(currency as char) as currency, - case when `date` = '' then NULL - else cast(`date` as date) - end as `date` - , - cast(nullif(timestamp_col, '') as char) as timestamp_col, - cast(`HKD@spéçiäl & characters` as - float -) as `HKD@spéçiäl & characters`, - cast(hkd_special___characters as char) as hkd_special___characters, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - _airbyte_emitted_at -from _airbyte_test_normalization.`dedup_exchange_rate_ab1` --- dedup_exchange_rate - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql deleted file mode 100644 index 83a9b9726049..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql +++ /dev/null @@ -1,10 +0,0 @@ - - create view _airbyte_test_normalization.`dedup_exchange_rate_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(timestamp_col as char), ''), '-', coalesce(cast(`HKD@spéçiäl & characters` as char), ''), '-', coalesce(cast(hkd_special___characters as char), ''), '-', coalesce(cast(nzd as char), ''), '-', coalesce(cast(usd as char), '')) as char)) as _airbyte_dedup_exchange_rate_hashid, - tmp.* -from _airbyte_test_normalization.`dedup_exchange_rate_ab2` tmp --- dedup_exchange_rate - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql deleted file mode 100644 index 6e7a324f9a3d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view _airbyte_test_normalization.`dedup_exchange_rate_ab4__dbt_tmp` as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_exchange_rate_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from _airbyte_test_normalization.`dedup_exchange_rate_ab3` tmp --- dedup_exchange_rate from test_normalization._airbyte_raw_dedup_exchange_rate - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab1.sql deleted file mode 100644 index b8662205d239..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ /dev/null @@ -1,25 +0,0 @@ - - create view _airbyte_test_normalization.`exchange_rate_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_value(_airbyte_data, - '$."currency"') as currency, - json_value(_airbyte_data, - '$."date"') as `date`, - json_value(_airbyte_data, - '$."timestamp_col"') as timestamp_col, - json_value(_airbyte_data, - '$."HKD@spéçiäl & characters"') as `HKD@spéçiäl & characters`, - json_value(_airbyte_data, - '$."HKD_special___characters"') as hkd_special___characters, - json_value(_airbyte_data, - '$."NZD"') as nzd, - json_value(_airbyte_data, - '$."USD"') as usd, - _airbyte_emitted_at -from test_normalization._airbyte_raw_exchange_rate as table_alias --- exchange_rate - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab2.sql deleted file mode 100644 index 081f20d75019..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ /dev/null @@ -1,28 +0,0 @@ - - create view _airbyte_test_normalization.`exchange_rate_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - signed -) as id, - cast(currency as char) as currency, - case when `date` = '' then NULL - else cast(`date` as date) - end as `date` - , - cast(nullif(timestamp_col, '') as char) as timestamp_col, - cast(`HKD@spéçiäl & characters` as - float -) as `HKD@spéçiäl & characters`, - cast(hkd_special___characters as char) as hkd_special___characters, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - _airbyte_emitted_at -from _airbyte_test_normalization.`exchange_rate_ab1` --- exchange_rate - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab3.sql deleted file mode 100644 index 85d7fb2288bc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ /dev/null @@ -1,10 +0,0 @@ - - create view _airbyte_test_normalization.`exchange_rate_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(timestamp_col as char), ''), '-', coalesce(cast(`HKD@spéçiäl & characters` as char), ''), '-', coalesce(cast(hkd_special___characters as char), ''), '-', coalesce(cast(nzd as char), ''), '-', coalesce(cast(usd as char), '')) as char)) as _airbyte_exchange_rate_hashid, - tmp.* -from _airbyte_test_normalization.`exchange_rate_ab2` tmp --- exchange_rate - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql deleted file mode 100644 index 16b2b48b4ca4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ /dev/null @@ -1,21 +0,0 @@ - - create view _airbyte_test_normalization.`pos_dedup_cdcx_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_value(_airbyte_data, - '$."name"') as `name`, - json_value(_airbyte_data, - '$."_ab_cdc_lsn"') as _ab_cdc_lsn, - json_value(_airbyte_data, - '$."_ab_cdc_updated_at"') as _ab_cdc_updated_at, - json_value(_airbyte_data, - '$."_ab_cdc_deleted_at"') as _ab_cdc_deleted_at, - json_value(_airbyte_data, - '$."_ab_cdc_log_pos"') as _ab_cdc_log_pos, - _airbyte_emitted_at -from test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias --- pos_dedup_cdcx - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql deleted file mode 100644 index 50eec775907d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ /dev/null @@ -1,25 +0,0 @@ - - create view _airbyte_test_normalization.`pos_dedup_cdcx_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - signed -) as id, - cast(`name` as char) as `name`, - cast(_ab_cdc_lsn as - float -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float -) as _ab_cdc_deleted_at, - cast(_ab_cdc_log_pos as - float -) as _ab_cdc_log_pos, - _airbyte_emitted_at -from _airbyte_test_normalization.`pos_dedup_cdcx_ab1` --- pos_dedup_cdcx - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index 6d71da37893e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,10 +0,0 @@ - - create view _airbyte_test_normalization.`pos_dedup_cdcx_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`name` as char), ''), '-', coalesce(cast(_ab_cdc_lsn as char), ''), '-', coalesce(cast(_ab_cdc_updated_at as char), ''), '-', coalesce(cast(_ab_cdc_deleted_at as char), ''), '-', coalesce(cast(_ab_cdc_log_pos as char), '')) as char)) as _airbyte_pos_dedup_cdcx_hashid, - tmp.* -from _airbyte_test_normalization.`pos_dedup_cdcx_ab2` tmp --- pos_dedup_cdcx - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql deleted file mode 100644 index f526a98321b9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view _airbyte_test_normalization.`pos_dedup_cdcx_ab4__dbt_tmp` as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_pos_dedup_cdcx_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from _airbyte_test_normalization.`pos_dedup_cdcx_ab3` tmp --- pos_dedup_cdcx from test_normalization._airbyte_raw_pos_dedup_cdcx - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index eb404ddd52a8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create table - test_normalization.`dedup_cdc_excluded__dbt_tmp` - as ( - --- Final base SQL model -select - id, - `name`, - `column__'with"_quotes`, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from test_normalization.`dedup_cdc_excluded_scd` --- dedup_cdc_excluded from test_normalization._airbyte_raw_dedup_cdc_excluded -where _airbyte_active_row = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index 900871e7e77a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,29 +0,0 @@ - - - create table - test_normalization.`dedup_cdc_excluded_scd__dbt_tmp` - as ( - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - `name`, - `column__'with"_quotes`, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from _airbyte_test_normalization.`dedup_cdc_excluded_ab4` --- dedup_cdc_excluded from test_normalization._airbyte_raw_dedup_cdc_excluded -where _airbyte_row_num = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql deleted file mode 100644 index d38a5c453ae5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql +++ /dev/null @@ -1,22 +0,0 @@ - - - create table - test_normalization.`dedup_exchange_rate__dbt_tmp` - as ( - --- Final base SQL model -select - id, - currency, - `date`, - timestamp_col, - `HKD@spéçiäl & characters`, - hkd_special___characters, - nzd, - usd, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from test_normalization.`dedup_exchange_rate_scd` --- dedup_exchange_rate from test_normalization._airbyte_raw_dedup_exchange_rate -where _airbyte_active_row = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql deleted file mode 100644 index 7376aaa375be..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql +++ /dev/null @@ -1,31 +0,0 @@ - - - create table - test_normalization.`dedup_exchange_rate_scd__dbt_tmp` - as ( - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - currency, - `date`, - timestamp_col, - `HKD@spéçiäl & characters`, - hkd_special___characters, - nzd, - usd, - `date` as _airbyte_start_at, - lag(`date`) over ( - partition by id, currency, cast(nzd as char) - order by `date` is null asc, `date` desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(`date`) over ( - partition by id, currency, cast(nzd as char) - order by `date` is null asc, `date` desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from _airbyte_test_normalization.`dedup_exchange_rate_ab4` --- dedup_exchange_rate from test_normalization._airbyte_raw_dedup_exchange_rate -where _airbyte_row_num = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql deleted file mode 100644 index e24cf0e7d279..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create table - test_normalization.`exchange_rate__dbt_tmp` - as ( - --- Final base SQL model -select - id, - currency, - `date`, - timestamp_col, - `HKD@spéçiäl & characters`, - hkd_special___characters, - nzd, - usd, - _airbyte_emitted_at, - _airbyte_exchange_rate_hashid -from _airbyte_test_normalization.`exchange_rate_ab3` --- exchange_rate from test_normalization._airbyte_raw_exchange_rate - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql deleted file mode 100644 index e166fc1d244e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create table - test_normalization.`pos_dedup_cdcx__dbt_tmp` - as ( - --- Final base SQL model -select - id, - `name`, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid -from test_normalization.`pos_dedup_cdcx_scd` --- pos_dedup_cdcx from test_normalization._airbyte_raw_pos_dedup_cdcx -where _airbyte_active_row = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index 9712b7bb96aa..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,29 +0,0 @@ - - - create table - test_normalization.`pos_dedup_cdcx_scd__dbt_tmp` - as ( - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - `name`, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid -from _airbyte_test_normalization.`pos_dedup_cdcx_ab4` --- pos_dedup_cdcx from test_normalization._airbyte_raw_pos_dedup_cdcx -where _airbyte_row_num = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..33bc1c9a474e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -0,0 +1,25 @@ + + create view _airbyte_test_normalization.`dedup_cdc_excluded_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."name"') as `name`, + json_value(_airbyte_data, + '$."_ab_cdc_lsn"') as _ab_cdc_lsn, + json_value(_airbyte_data, + '$."_ab_cdc_updated_at"') as _ab_cdc_updated_at, + json_value(_airbyte_data, + '$."_ab_cdc_deleted_at"') as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias +-- dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..c725452b87b1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -0,0 +1,28 @@ + + create view _airbyte_test_normalization.`dedup_cdc_excluded_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + signed +) as id, + cast(`name` as char) as `name`, + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`dedup_cdc_excluded_ab1` +-- dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql new file mode 100644 index 000000000000..e42ac40db163 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -0,0 +1,31 @@ + + create view _airbyte_test_normalization.`dedup_exchange_rate_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."currency"') as currency, + json_value(_airbyte_data, + '$."date"') as `date`, + json_value(_airbyte_data, + '$."timestamp_col"') as timestamp_col, + json_value(_airbyte_data, + '$."HKD@spéçiäl & characters"') as `HKD@spéçiäl & characters`, + json_value(_airbyte_data, + '$."HKD_special___characters"') as hkd_special___characters, + json_value(_airbyte_data, + '$."NZD"') as nzd, + json_value(_airbyte_data, + '$."USD"') as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql new file mode 100644 index 000000000000..489196399610 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -0,0 +1,34 @@ + + create view _airbyte_test_normalization.`dedup_exchange_rate_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + signed +) as id, + cast(currency as char) as currency, + case when `date` = '' then NULL + else cast(`date` as date) + end as `date` + , + cast(nullif(timestamp_col, '') as char) as timestamp_col, + cast(`HKD@spéçiäl & characters` as + float +) as `HKD@spéçiäl & characters`, + cast(hkd_special___characters as char) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`dedup_exchange_rate_ab1` +-- dedup_exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql new file mode 100644 index 000000000000..335aa00cf89f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -0,0 +1,31 @@ + + create view _airbyte_test_normalization.`exchange_rate_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."currency"') as currency, + json_value(_airbyte_data, + '$."date"') as `date`, + json_value(_airbyte_data, + '$."timestamp_col"') as timestamp_col, + json_value(_airbyte_data, + '$."HKD@spéçiäl & characters"') as `HKD@spéçiäl & characters`, + json_value(_airbyte_data, + '$."HKD_special___characters"') as hkd_special___characters, + json_value(_airbyte_data, + '$."NZD"') as nzd, + json_value(_airbyte_data, + '$."USD"') as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql new file mode 100644 index 000000000000..258e3f4caf08 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -0,0 +1,34 @@ + + create view _airbyte_test_normalization.`exchange_rate_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + signed +) as id, + cast(currency as char) as currency, + case when `date` = '' then NULL + else cast(`date` as date) + end as `date` + , + cast(nullif(timestamp_col, '') as char) as timestamp_col, + cast(`HKD@spéçiäl & characters` as + float +) as `HKD@spéçiäl & characters`, + cast(hkd_special___characters as char) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`exchange_rate_ab1` +-- exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql new file mode 100644 index 000000000000..6cc3a26b097f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -0,0 +1,12 @@ + + create view _airbyte_test_normalization.`exchange_rate_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(timestamp_col as char), ''), '-', coalesce(cast(`HKD@spéçiäl & characters` as char), ''), '-', coalesce(cast(hkd_special___characters as char), ''), '-', coalesce(cast(nzd as char), ''), '-', coalesce(cast(usd as char), '')) as char)) as _airbyte_exchange_rate_hashid, + tmp.* +from _airbyte_test_normalization.`exchange_rate_ab2` tmp +-- exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql new file mode 100644 index 000000000000..7fd41343ac43 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -0,0 +1,26 @@ + + create view _airbyte_test_normalization.`pos_dedup_cdcx_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."name"') as `name`, + json_value(_airbyte_data, + '$."_ab_cdc_lsn"') as _ab_cdc_lsn, + json_value(_airbyte_data, + '$."_ab_cdc_updated_at"') as _ab_cdc_updated_at, + json_value(_airbyte_data, + '$."_ab_cdc_deleted_at"') as _ab_cdc_deleted_at, + json_value(_airbyte_data, + '$."_ab_cdc_log_pos"') as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias +-- pos_dedup_cdcx +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql new file mode 100644 index 000000000000..9f8c55f05dcf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -0,0 +1,30 @@ + + create view _airbyte_test_normalization.`pos_dedup_cdcx_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + signed +) as id, + cast(`name` as char) as `name`, + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + cast(_ab_cdc_log_pos as + float +) as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`pos_dedup_cdcx_ab1` +-- pos_dedup_cdcx +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..446b1c167a26 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,17 @@ + + create view _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..01fe01259f91 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + signed +) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1` +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..4dd4da0f3431 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,74 @@ + + + create table + test_normalization.`dedup_cdc_excluded_scd__dbt_tmp` + as ( + +with + +input_data as ( + select * + from _airbyte_test_normalization.`dedup_cdc_excluded_ab3` + -- dedup_cdc_excluded from test_normalization._airbyte_raw_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_unique_key, + id, + `name`, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as char), cast(_ab_cdc_updated_at as char) + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), ''), '-', coalesce(cast(_ab_cdc_deleted_at as char), ''), '-', coalesce(cast(_ab_cdc_updated_at as char), '')) as char)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + `name`, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..03eb062ccd38 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,80 @@ + + + create table + test_normalization.`dedup_exchange_rate_scd__dbt_tmp` + as ( + +with + +input_data as ( + select * + from _airbyte_test_normalization.`dedup_exchange_rate_ab3` + -- dedup_exchange_rate from test_normalization._airbyte_raw_dedup_exchange_rate +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(nzd as char), '')) as char)) as _airbyte_unique_key, + id, + currency, + `date`, + timestamp_col, + `HKD@spéçiäl & characters`, + hkd_special___characters, + nzd, + usd, + `date` as _airbyte_start_at, + lag(`date`) over ( + partition by id, currency, cast(nzd as char) + order by + `date` is null asc, + `date` desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(`date`) over ( + partition by id, currency, cast(nzd as char) + order by + `date` is null asc, + `date` desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_exchange_rate_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), '')) as char)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + currency, + `date`, + timestamp_col, + `HKD@spéçiäl & characters`, + hkd_special___characters, + nzd, + usd, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from dedup_data where _airbyte_row_num = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..13bbe0a227a9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,66 @@ + + + create table + test_normalization.`renamed_dedup_cdc_excluded_scd__dbt_tmp` + as ( + +with + +input_data as ( + select * + from _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab3` + -- renamed_dedup_cdc_excluded from test_normalization._airbyte_raw_renamed_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_unique_key, + id, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_renamed_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), '')) as char)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql new file mode 100644 index 000000000000..37503d5bb636 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -0,0 +1,26 @@ + + + create table + test_normalization.`dedup_cdc_excluded__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + `name`, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from test_normalization.`dedup_cdc_excluded_scd` +-- dedup_cdc_excluded from test_normalization._airbyte_raw_dedup_cdc_excluded +where 1 = 1 +and _airbyte_active_row = 1 + + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..c5ad30dc78c0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,29 @@ + + + create table + test_normalization.`dedup_exchange_rate__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + currency, + `date`, + timestamp_col, + `HKD@spéçiäl & characters`, + hkd_special___characters, + nzd, + usd, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from test_normalization.`dedup_exchange_rate_scd` +-- dedup_exchange_rate from test_normalization._airbyte_raw_dedup_exchange_rate +where 1 = 1 +and _airbyte_active_row = 1 + + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..6b8a3751184f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -0,0 +1,27 @@ + + + create table + test_normalization.`exchange_rate__dbt_tmp` + as ( + +-- Final base SQL model +select + id, + currency, + `date`, + timestamp_col, + `HKD@spéçiäl & characters`, + hkd_special___characters, + nzd, + usd, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from _airbyte_test_normalization.`exchange_rate_ab3` +-- exchange_rate from test_normalization._airbyte_raw_exchange_rate +where 1 = 1 + + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..10a547a0cccb --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,22 @@ + + + create table + test_normalization.`renamed_dedup_cdc_excluded__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from test_normalization.`renamed_dedup_cdc_excluded_scd` +-- renamed_dedup_cdc_excluded from test_normalization._airbyte_raw_renamed_dedup_cdc_excluded +where 1 = 1 +and _airbyte_active_row = 1 + + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..be6a9a952a26 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,76 @@ + + + create table + test_normalization.`pos_dedup_cdcx_scd__dbt_tmp` + as ( + +with + +input_data as ( + select * + from _airbyte_test_normalization.`pos_dedup_cdcx_ab3` + -- pos_dedup_cdcx from test_normalization._airbyte_raw_pos_dedup_cdcx +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_unique_key, + id, + `name`, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as char), cast(_ab_cdc_updated_at as char), cast(_ab_cdc_log_pos as char) + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), ''), '-', coalesce(cast(_ab_cdc_deleted_at as char), ''), '-', coalesce(cast(_ab_cdc_updated_at as char), ''), '-', coalesce(cast(_ab_cdc_log_pos as char), '')) as char)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + `name`, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql new file mode 100644 index 000000000000..736d35c45a9a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -0,0 +1,26 @@ + + + create table + test_normalization.`pos_dedup_cdcx__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + `name`, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from test_normalization.`pos_dedup_cdcx_scd` +-- pos_dedup_cdcx from test_normalization._airbyte_raw_pos_dedup_cdcx +where 1 = 1 +and _airbyte_active_row = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..3caccb347235 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,12 @@ + + create view _airbyte_test_normalization.`dedup_cdc_excluded_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`name` as char), ''), '-', coalesce(cast(_ab_cdc_lsn as char), ''), '-', coalesce(cast(_ab_cdc_updated_at as char), ''), '-', coalesce(cast(_ab_cdc_deleted_at as char), '')) as char)) as _airbyte_dedup_cdc_excluded_hashid, + tmp.* +from _airbyte_test_normalization.`dedup_cdc_excluded_ab2` tmp +-- dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..98129db90185 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,12 @@ + + create view _airbyte_test_normalization.`dedup_exchange_rate_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(timestamp_col as char), ''), '-', coalesce(cast(`HKD@spéçiäl & characters` as char), ''), '-', coalesce(cast(hkd_special___characters as char), ''), '-', coalesce(cast(nzd as char), ''), '-', coalesce(cast(usd as char), '')) as char)) as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from _airbyte_test_normalization.`dedup_exchange_rate_ab2` tmp +-- dedup_exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql new file mode 100644 index 000000000000..857c87e6fc1d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`pos_dedup_cdcx_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`name` as char), ''), '-', coalesce(cast(_ab_cdc_lsn as char), ''), '-', coalesce(cast(_ab_cdc_updated_at as char), ''), '-', coalesce(cast(_ab_cdc_deleted_at as char), ''), '-', coalesce(cast(_ab_cdc_log_pos as char), '')) as char)) as _airbyte_pos_dedup_cdcx_hashid, + tmp.* +from _airbyte_test_normalization.`pos_dedup_cdcx_ab2` tmp +-- pos_dedup_cdcx +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..2df6fcdc3715 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,12 @@ + + create view _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2` tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql deleted file mode 100644 index 6144a13617e8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql +++ /dev/null @@ -1,43 +0,0 @@ -with table_row_counts as ( - select distinct count(*) as row_count, 10 as expected_count - from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} -union all - select distinct count(*) as row_count, 10 as expected_count - from {{ ref('exchange_rate') }} - -union all - - select distinct count(*) as row_count, 10 as expected_count - from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -union all - select distinct count(*) as row_count, 10 as expected_count - from {{ ref('dedup_exchange_rate_scd') }} -union all - select distinct count(*) as row_count, 5 as expected_count - from {{ ref('dedup_exchange_rate') }} - -union all - - select distinct count(*) as row_count, 8 as expected_count - from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -union all - select distinct count(*) as row_count, 8 as expected_count - from {{ ref('dedup_cdc_excluded_scd') }} -union all - select distinct count(*) as row_count, 4 as expected_count - from {{ ref('dedup_cdc_excluded') }} - -union all - - select distinct count(*) as row_count, 8 as expected_count - from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -union all - select distinct count(*) as row_count, 8 as expected_count - from {{ ref('pos_dedup_cdcx_scd') }} -union all - select distinct count(*) as row_count, 3 as expected_count - from {{ ref('pos_dedup_cdcx') }} -) -select * -from table_row_counts -where row_count != expected_count diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql index 6d694c280ee7..5fb29052e85b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -1,13 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['name'], ['name']) }} as {{ adapter.quote('name') }}, - {{ json_extract_scalar('_airbyte_data', ['column`_\'with"_quotes'], ['column___with__quotes']) }} as {{ adapter.quote('column__\'with"_quotes') }}, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_lsn'], ['_ab_cdc_lsn']) }} as _ab_cdc_lsn, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} as table_alias -- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql index dc225106de96..42bbbbc98de8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -1,13 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, cast({{ adapter.quote('name') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('name') }}, - cast({{ adapter.quote('column__\'with"_quotes') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('column__\'with"_quotes') }}, cast(_ab_cdc_lsn as {{ dbt_utils.type_float() }}) as _ab_cdc_lsn, cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('dedup_cdc_excluded_ab1') }} -- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 67de7e9893fb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - adapter.quote('name'), - adapter.quote('column__\'with"_quotes'), - '_ab_cdc_lsn', - '_ab_cdc_updated_at', - '_ab_cdc_deleted_at', - ]) }} as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from {{ ref('dedup_cdc_excluded_ab2') }} tmp --- dedup_cdc_excluded - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql deleted file mode 100644 index aa34179c3c5c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_cdc_excluded_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('dedup_cdc_excluded_ab3') }} tmp --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql index c9a6865ef7fd..5ed2b660ee09 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, @@ -9,7 +13,11 @@ select {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as hkd_special___characters, {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as nzd, {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} as table_alias -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index 4690ff9a33f1..a45b5767c899 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -12,7 +16,11 @@ select cast(hkd_special___characters as {{ dbt_utils.type_string() }}) as hkd_special___characters, cast(nzd as {{ dbt_utils.type_float() }}) as nzd, cast(usd as {{ dbt_utils.type_float() }}) as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('dedup_exchange_rate_ab1') }} -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql deleted file mode 100644 index 3e68860804dc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'currency', - adapter.quote('date'), - 'timestamp_col', - adapter.quote('HKD@spéçiäl & characters'), - 'hkd_special___characters', - 'nzd', - 'usd', - ]) }} as _airbyte_dedup_exchange_rate_hashid, - tmp.* -from {{ ref('dedup_exchange_rate_ab2') }} tmp --- dedup_exchange_rate - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql deleted file mode 100644 index ef5f6e52d60e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_exchange_rate_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('dedup_exchange_rate_ab3') }} tmp --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql index 3e66f6d0b4fc..c20f382cfbc0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, @@ -9,7 +13,11 @@ select {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as hkd_special___characters, {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as nzd, {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} as table_alias -- exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql index de09dc96002c..733803bde010 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -12,7 +16,11 @@ select cast(hkd_special___characters as {{ dbt_utils.type_string() }}) as hkd_special___characters, cast(nzd as {{ dbt_utils.type_float() }}) as nzd, cast(usd as {{ dbt_utils.type_float() }}) as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('exchange_rate_ab1') }} -- exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql index 962cb57a8666..c645234a8db9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -14,4 +18,6 @@ select tmp.* from {{ ref('exchange_rate_ab2') }} tmp -- exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql index 1bd874551007..ac502fab3a57 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, @@ -7,7 +11,10 @@ select {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_log_pos'], ['_ab_cdc_log_pos']) }} as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} as table_alias -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql index c132580004d5..f6664f48d0bb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -7,7 +11,10 @@ select cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, cast(_ab_cdc_log_pos as {{ dbt_utils.type_float() }}) as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('pos_dedup_cdcx_ab1') }} -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index 5b9a513b2e86..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - adapter.quote('name'), - '_ab_cdc_lsn', - '_ab_cdc_updated_at', - '_ab_cdc_deleted_at', - '_ab_cdc_log_pos', - ]) }} as _airbyte_pos_dedup_cdcx_hashid, - tmp.* -from {{ ref('pos_dedup_cdcx_ab2') }} tmp --- pos_dedup_cdcx - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql deleted file mode 100644 index 7c897124b395..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_pos_dedup_cdcx_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('pos_dedup_cdcx_ab3') }} tmp --- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..7fd3046082a7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,16 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..a04f92bdcd55 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,16 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as {{ dbt_utils.type_bigint() }}) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ ref('renamed_dedup_cdc_excluded_ab1') }} +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..ff4320b54f64 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,111 @@ +{{ config( + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('dedup_cdc_excluded_ab3') }} + -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('dedup_cdc_excluded_ab3') }} + -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}) + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..470401fb5740 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,121 @@ +{{ config( + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('dedup_exchange_rate_ab3') }} + -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + 'currency', + 'nzd', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('dedup_exchange_rate_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('dedup_exchange_rate_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('dedup_exchange_rate_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('dedup_exchange_rate_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('dedup_exchange_rate_ab3') }} + -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + 'currency', + 'nzd', + ]) }} as _airbyte_unique_key, + id, + currency, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + {{ adapter.quote('date') }} as _airbyte_start_at, + lag({{ adapter.quote('date') }}) over ( + partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) + order by + {{ adapter.quote('date') }} is null asc, + {{ adapter.quote('date') }} desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag({{ adapter.quote('date') }}) over ( + partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) + order by + {{ adapter.quote('date') }} is null asc, + {{ adapter.quote('date') }} desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_exchange_rate_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + currency, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..b4ae7c90fe73 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,103 @@ +{{ config( + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('renamed_dedup_cdc_excluded_ab3') }} + -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('renamed_dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('renamed_dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('renamed_dedup_cdc_excluded_ab3') }} + -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_renamed_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql new file mode 100644 index 000000000000..af2a08ea048f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -0,0 +1,23 @@ +{{ config( + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + id, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from {{ ref('dedup_cdc_excluded_scd') }} +-- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..06a37a760378 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,26 @@ +{{ config( + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + id, + currency, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from {{ ref('dedup_exchange_rate_scd') }} +-- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..980126a4146c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql @@ -0,0 +1,24 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + id, + currency, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from {{ ref('exchange_rate_ab3') }} +-- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..8a8ff85f5902 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,19 @@ +{{ config( + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from {{ ref('renamed_dedup_cdc_excluded_scd') }} +-- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..7a3a501bb98f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,113 @@ +{{ config( + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('pos_dedup_cdcx_ab3') }} + -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('pos_dedup_cdcx_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('pos_dedup_cdcx_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('pos_dedup_cdcx_ab3') }} + -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_log_pos as {{ dbt_utils.type_string() }}) + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at', '_ab_cdc_log_pos' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index 2152d113d03a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - {{ adapter.quote('name') }}, - {{ adapter.quote('column__\'with"_quotes') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from {{ ref('dedup_cdc_excluded_scd') }} --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -where _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index 8d417147dfdd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - {{ adapter.quote('name') }}, - {{ adapter.quote('column__\'with"_quotes') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from {{ ref('dedup_cdc_excluded_ab4') }} --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql deleted file mode 100644 index f10a9446d7b8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - currency, - {{ adapter.quote('date') }}, - timestamp_col, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from {{ ref('dedup_exchange_rate_scd') }} --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -where _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql deleted file mode 100644 index 739992eb7d2e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql +++ /dev/null @@ -1,26 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - currency, - {{ adapter.quote('date') }}, - timestamp_col, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, - {{ adapter.quote('date') }} as _airbyte_start_at, - lag({{ adapter.quote('date') }}) over ( - partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) - order by {{ adapter.quote('date') }} is null asc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag({{ adapter.quote('date') }}) over ( - partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) - order by {{ adapter.quote('date') }} is null asc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from {{ ref('dedup_exchange_rate_ab4') }} --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql deleted file mode 100644 index 147399e6739d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - currency, - {{ adapter.quote('date') }}, - timestamp_col, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, - _airbyte_emitted_at, - _airbyte_exchange_rate_hashid -from {{ ref('exchange_rate_ab3') }} --- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql index ae62dc05f870..1a756d5b7507 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -1,15 +1,23 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select + _airbyte_unique_key, id, {{ adapter.quote('name') }}, _ab_cdc_lsn, _ab_cdc_updated_at, _ab_cdc_deleted_at, _ab_cdc_log_pos, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_pos_dedup_cdcx_hashid from {{ ref('pos_dedup_cdcx_scd') }} -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index c25354e4fba4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - {{ adapter.quote('name') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid -from {{ ref('pos_dedup_cdcx_ab4') }} --- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..6d34fc802057 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,20 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + adapter.quote('name'), + '_ab_cdc_lsn', + '_ab_cdc_updated_at', + '_ab_cdc_deleted_at', + ]) }} as _airbyte_dedup_cdc_excluded_hashid, + tmp.* +from {{ ref('dedup_cdc_excluded_ab2') }} tmp +-- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..9221c1833031 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,23 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + 'currency', + adapter.quote('date'), + 'timestamp_col', + adapter.quote('HKD@spéçiäl & characters'), + 'hkd_special___characters', + 'nzd', + 'usd', + ]) }} as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from {{ ref('dedup_exchange_rate_ab2') }} tmp +-- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql new file mode 100644 index 000000000000..78def1d5d1aa --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -0,0 +1,20 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + adapter.quote('name'), + '_ab_cdc_lsn', + '_ab_cdc_updated_at', + '_ab_cdc_deleted_at', + '_ab_cdc_log_pos', + ]) }} as _airbyte_pos_dedup_cdcx_hashid, + tmp.* +from {{ ref('pos_dedup_cdcx_ab2') }} tmp +-- pos_dedup_cdcx +where 1 = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..f5ce3e6ba182 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,16 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from {{ ref('renamed_dedup_cdc_excluded_ab2') }} tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/sources.yml index c085ca8ccc25..45c338b893ca 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/sources.yml @@ -10,3 +10,4 @@ sources: - name: _airbyte_raw_dedup_exchange_rate - name: _airbyte_raw_exchange_rate - name: _airbyte_raw_pos_dedup_cdcx + - name: _airbyte_raw_renamed_dedup_cdc_excluded diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..33bc1c9a474e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -0,0 +1,25 @@ + + create view _airbyte_test_normalization.`dedup_cdc_excluded_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."name"') as `name`, + json_value(_airbyte_data, + '$."_ab_cdc_lsn"') as _ab_cdc_lsn, + json_value(_airbyte_data, + '$."_ab_cdc_updated_at"') as _ab_cdc_updated_at, + json_value(_airbyte_data, + '$."_ab_cdc_deleted_at"') as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias +-- dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..c725452b87b1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -0,0 +1,28 @@ + + create view _airbyte_test_normalization.`dedup_cdc_excluded_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + signed +) as id, + cast(`name` as char) as `name`, + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`dedup_cdc_excluded_ab1` +-- dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql new file mode 100644 index 000000000000..e42ac40db163 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -0,0 +1,31 @@ + + create view _airbyte_test_normalization.`dedup_exchange_rate_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."currency"') as currency, + json_value(_airbyte_data, + '$."date"') as `date`, + json_value(_airbyte_data, + '$."timestamp_col"') as timestamp_col, + json_value(_airbyte_data, + '$."HKD@spéçiäl & characters"') as `HKD@spéçiäl & characters`, + json_value(_airbyte_data, + '$."HKD_special___characters"') as hkd_special___characters, + json_value(_airbyte_data, + '$."NZD"') as nzd, + json_value(_airbyte_data, + '$."USD"') as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql new file mode 100644 index 000000000000..489196399610 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -0,0 +1,34 @@ + + create view _airbyte_test_normalization.`dedup_exchange_rate_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + signed +) as id, + cast(currency as char) as currency, + case when `date` = '' then NULL + else cast(`date` as date) + end as `date` + , + cast(nullif(timestamp_col, '') as char) as timestamp_col, + cast(`HKD@spéçiäl & characters` as + float +) as `HKD@spéçiäl & characters`, + cast(hkd_special___characters as char) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`dedup_exchange_rate_ab1` +-- dedup_exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql new file mode 100644 index 000000000000..335aa00cf89f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -0,0 +1,31 @@ + + create view _airbyte_test_normalization.`exchange_rate_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."currency"') as currency, + json_value(_airbyte_data, + '$."date"') as `date`, + json_value(_airbyte_data, + '$."timestamp_col"') as timestamp_col, + json_value(_airbyte_data, + '$."HKD@spéçiäl & characters"') as `HKD@spéçiäl & characters`, + json_value(_airbyte_data, + '$."HKD_special___characters"') as hkd_special___characters, + json_value(_airbyte_data, + '$."NZD"') as nzd, + json_value(_airbyte_data, + '$."USD"') as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql new file mode 100644 index 000000000000..258e3f4caf08 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -0,0 +1,34 @@ + + create view _airbyte_test_normalization.`exchange_rate_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + signed +) as id, + cast(currency as char) as currency, + case when `date` = '' then NULL + else cast(`date` as date) + end as `date` + , + cast(nullif(timestamp_col, '') as char) as timestamp_col, + cast(`HKD@spéçiäl & characters` as + float +) as `HKD@spéçiäl & characters`, + cast(hkd_special___characters as char) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`exchange_rate_ab1` +-- exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql new file mode 100644 index 000000000000..6cc3a26b097f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -0,0 +1,12 @@ + + create view _airbyte_test_normalization.`exchange_rate_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(timestamp_col as char), ''), '-', coalesce(cast(`HKD@spéçiäl & characters` as char), ''), '-', coalesce(cast(hkd_special___characters as char), ''), '-', coalesce(cast(nzd as char), ''), '-', coalesce(cast(usd as char), '')) as char)) as _airbyte_exchange_rate_hashid, + tmp.* +from _airbyte_test_normalization.`exchange_rate_ab2` tmp +-- exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql new file mode 100644 index 000000000000..7fd41343ac43 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -0,0 +1,26 @@ + + create view _airbyte_test_normalization.`pos_dedup_cdcx_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."name"') as `name`, + json_value(_airbyte_data, + '$."_ab_cdc_lsn"') as _ab_cdc_lsn, + json_value(_airbyte_data, + '$."_ab_cdc_updated_at"') as _ab_cdc_updated_at, + json_value(_airbyte_data, + '$."_ab_cdc_deleted_at"') as _ab_cdc_deleted_at, + json_value(_airbyte_data, + '$."_ab_cdc_log_pos"') as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias +-- pos_dedup_cdcx +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql new file mode 100644 index 000000000000..9f8c55f05dcf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -0,0 +1,30 @@ + + create view _airbyte_test_normalization.`pos_dedup_cdcx_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + signed +) as id, + cast(`name` as char) as `name`, + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + cast(_ab_cdc_log_pos as + float +) as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`pos_dedup_cdcx_ab1` +-- pos_dedup_cdcx +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..446b1c167a26 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,17 @@ + + create view _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..01fe01259f91 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + signed +) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1` +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..4dd4da0f3431 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,74 @@ + + + create table + test_normalization.`dedup_cdc_excluded_scd__dbt_tmp` + as ( + +with + +input_data as ( + select * + from _airbyte_test_normalization.`dedup_cdc_excluded_ab3` + -- dedup_cdc_excluded from test_normalization._airbyte_raw_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_unique_key, + id, + `name`, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as char), cast(_ab_cdc_updated_at as char) + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), ''), '-', coalesce(cast(_ab_cdc_deleted_at as char), ''), '-', coalesce(cast(_ab_cdc_updated_at as char), '')) as char)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + `name`, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..03eb062ccd38 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,80 @@ + + + create table + test_normalization.`dedup_exchange_rate_scd__dbt_tmp` + as ( + +with + +input_data as ( + select * + from _airbyte_test_normalization.`dedup_exchange_rate_ab3` + -- dedup_exchange_rate from test_normalization._airbyte_raw_dedup_exchange_rate +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(nzd as char), '')) as char)) as _airbyte_unique_key, + id, + currency, + `date`, + timestamp_col, + `HKD@spéçiäl & characters`, + hkd_special___characters, + nzd, + usd, + `date` as _airbyte_start_at, + lag(`date`) over ( + partition by id, currency, cast(nzd as char) + order by + `date` is null asc, + `date` desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(`date`) over ( + partition by id, currency, cast(nzd as char) + order by + `date` is null asc, + `date` desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_exchange_rate_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), '')) as char)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + currency, + `date`, + timestamp_col, + `HKD@spéçiäl & characters`, + hkd_special___characters, + nzd, + usd, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from dedup_data where _airbyte_row_num = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..13bbe0a227a9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,66 @@ + + + create table + test_normalization.`renamed_dedup_cdc_excluded_scd__dbt_tmp` + as ( + +with + +input_data as ( + select * + from _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab3` + -- renamed_dedup_cdc_excluded from test_normalization._airbyte_raw_renamed_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_unique_key, + id, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_renamed_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), '')) as char)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql new file mode 100644 index 000000000000..37503d5bb636 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -0,0 +1,26 @@ + + + create table + test_normalization.`dedup_cdc_excluded__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + `name`, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from test_normalization.`dedup_cdc_excluded_scd` +-- dedup_cdc_excluded from test_normalization._airbyte_raw_dedup_cdc_excluded +where 1 = 1 +and _airbyte_active_row = 1 + + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..c5ad30dc78c0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,29 @@ + + + create table + test_normalization.`dedup_exchange_rate__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + currency, + `date`, + timestamp_col, + `HKD@spéçiäl & characters`, + hkd_special___characters, + nzd, + usd, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from test_normalization.`dedup_exchange_rate_scd` +-- dedup_exchange_rate from test_normalization._airbyte_raw_dedup_exchange_rate +where 1 = 1 +and _airbyte_active_row = 1 + + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..6b8a3751184f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -0,0 +1,27 @@ + + + create table + test_normalization.`exchange_rate__dbt_tmp` + as ( + +-- Final base SQL model +select + id, + currency, + `date`, + timestamp_col, + `HKD@spéçiäl & characters`, + hkd_special___characters, + nzd, + usd, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from _airbyte_test_normalization.`exchange_rate_ab3` +-- exchange_rate from test_normalization._airbyte_raw_exchange_rate +where 1 = 1 + + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..10a547a0cccb --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,22 @@ + + + create table + test_normalization.`renamed_dedup_cdc_excluded__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from test_normalization.`renamed_dedup_cdc_excluded_scd` +-- renamed_dedup_cdc_excluded from test_normalization._airbyte_raw_renamed_dedup_cdc_excluded +where 1 = 1 +and _airbyte_active_row = 1 + + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..be6a9a952a26 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,76 @@ + + + create table + test_normalization.`pos_dedup_cdcx_scd__dbt_tmp` + as ( + +with + +input_data as ( + select * + from _airbyte_test_normalization.`pos_dedup_cdcx_ab3` + -- pos_dedup_cdcx from test_normalization._airbyte_raw_pos_dedup_cdcx +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_unique_key, + id, + `name`, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as char), cast(_ab_cdc_updated_at as char), cast(_ab_cdc_log_pos as char) + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), ''), '-', coalesce(cast(_ab_cdc_deleted_at as char), ''), '-', coalesce(cast(_ab_cdc_updated_at as char), ''), '-', coalesce(cast(_ab_cdc_log_pos as char), '')) as char)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + `name`, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql new file mode 100644 index 000000000000..736d35c45a9a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -0,0 +1,26 @@ + + + create table + test_normalization.`pos_dedup_cdcx__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + `name`, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from test_normalization.`pos_dedup_cdcx_scd` +-- pos_dedup_cdcx from test_normalization._airbyte_raw_pos_dedup_cdcx +where 1 = 1 +and _airbyte_active_row = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..3caccb347235 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,12 @@ + + create view _airbyte_test_normalization.`dedup_cdc_excluded_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`name` as char), ''), '-', coalesce(cast(_ab_cdc_lsn as char), ''), '-', coalesce(cast(_ab_cdc_updated_at as char), ''), '-', coalesce(cast(_ab_cdc_deleted_at as char), '')) as char)) as _airbyte_dedup_cdc_excluded_hashid, + tmp.* +from _airbyte_test_normalization.`dedup_cdc_excluded_ab2` tmp +-- dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..98129db90185 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,12 @@ + + create view _airbyte_test_normalization.`dedup_exchange_rate_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(timestamp_col as char), ''), '-', coalesce(cast(`HKD@spéçiäl & characters` as char), ''), '-', coalesce(cast(hkd_special___characters as char), ''), '-', coalesce(cast(nzd as char), ''), '-', coalesce(cast(usd as char), '')) as char)) as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from _airbyte_test_normalization.`dedup_exchange_rate_ab2` tmp +-- dedup_exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql new file mode 100644 index 000000000000..857c87e6fc1d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`pos_dedup_cdcx_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`name` as char), ''), '-', coalesce(cast(_ab_cdc_lsn as char), ''), '-', coalesce(cast(_ab_cdc_updated_at as char), ''), '-', coalesce(cast(_ab_cdc_deleted_at as char), ''), '-', coalesce(cast(_ab_cdc_log_pos as char), '')) as char)) as _airbyte_pos_dedup_cdcx_hashid, + tmp.* +from _airbyte_test_normalization.`pos_dedup_cdcx_ab2` tmp +-- pos_dedup_cdcx +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..2df6fcdc3715 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,12 @@ + + create view _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2` tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index d55edca65a1f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create table test_normalization.dedup_cdc_excluded__dbt_tmp - - as - --- Final base SQL model -select - id, - name, - column___with__quotes, - "_AB_CDC_LSN", - "_AB_CDC_UPDATED_AT", - "_AB_CDC_DELETED_AT", - "_AIRBYTE_EMITTED_AT", - "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" -from test_normalization.dedup_cdc_excluded_scd --- dedup_cdc_excluded from test_normalization.airbyte_raw_dedup_cdc_excluded -where "_AIRBYTE_ACTIVE_ROW" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index 14f757f19bcf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,100 +0,0 @@ - - - create table test_normalization.dedup_cdc_excluded_scd__dbt_tmp - - as - -with dbt__cte__dedup_cdc_excluded_ab1__ as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value("_AIRBYTE_DATA", '$."id"') as id, - json_value("_AIRBYTE_DATA", '$."name"') as name, - json_value("_AIRBYTE_DATA", '$."column___with__quotes"') as column___with__quotes, - json_value("_AIRBYTE_DATA", '$."_ab_cdc_lsn"') as "_AB_CDC_LSN", - json_value("_AIRBYTE_DATA", '$."_ab_cdc_updated_at"') as "_AB_CDC_UPDATED_AT", - json_value("_AIRBYTE_DATA", '$."_ab_cdc_deleted_at"') as "_AB_CDC_DELETED_AT", - "_AIRBYTE_EMITTED_AT" -from test_normalization.airbyte_raw_dedup_cdc_excluded --- dedup_cdc_excluded -), dbt__cte__dedup_cdc_excluded_ab2__ as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - numeric -) as id, - cast(name as varchar2(4000)) as name, - cast(column___with__quotes as varchar2(4000)) as column___with__quotes, - cast("_AB_CDC_LSN" as - float -) as "_AB_CDC_LSN", - cast("_AB_CDC_UPDATED_AT" as - float -) as "_AB_CDC_UPDATED_AT", - cast("_AB_CDC_DELETED_AT" as - float -) as "_AB_CDC_DELETED_AT", - "_AIRBYTE_EMITTED_AT" -from dbt__cte__dedup_cdc_excluded_ab1__ --- dedup_cdc_excluded -), dbt__cte__dedup_cdc_excluded_ab3__ as ( - --- SQL model to build a hash column based on the values of this record -select - ora_hash( - - id || '~' || - - - name || '~' || - - - column___with__quotes || '~' || - - - "_AB_CDC_LSN" || '~' || - - - "_AB_CDC_UPDATED_AT" || '~' || - - - "_AB_CDC_DELETED_AT" - - ) as "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID", - tmp.* -from dbt__cte__dedup_cdc_excluded_ab2__ tmp --- dedup_cdc_excluded -), dbt__cte__dedup_cdc_excluded_ab4__ as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" - order by "_AIRBYTE_EMITTED_AT" asc - ) as "_AIRBYTE_ROW_NUM", - tmp.* -from dbt__cte__dedup_cdc_excluded_ab3__ tmp --- dedup_cdc_excluded from test_normalization.airbyte_raw_dedup_cdc_excluded -)-- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - column___with__quotes, - "_AB_CDC_LSN", - "_AB_CDC_UPDATED_AT", - "_AB_CDC_DELETED_AT", - "_AIRBYTE_EMITTED_AT" as "_AIRBYTE_START_AT", - lag("_AIRBYTE_EMITTED_AT") over ( - partition by id - order by "_AIRBYTE_EMITTED_AT" asc nulls last, "_AIRBYTE_EMITTED_AT" desc, "_AIRBYTE_EMITTED_AT" desc - ) as "_AIRBYTE_END_AT", - case when lag("_AIRBYTE_EMITTED_AT") over ( - partition by id - order by "_AIRBYTE_EMITTED_AT" asc nulls last, "_AIRBYTE_EMITTED_AT" desc, "_AIRBYTE_EMITTED_AT" desc, "_AB_CDC_UPDATED_AT" desc - ) is null and "_AB_CDC_DELETED_AT" is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", - "_AIRBYTE_EMITTED_AT", - "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" -from dbt__cte__dedup_cdc_excluded_ab4__ --- dedup_cdc_excluded from test_normalization.airbyte_raw_dedup_cdc_excluded -where "_AIRBYTE_ROW_NUM" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql deleted file mode 100644 index 72b753fb7e87..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create table test_normalization.dedup_exchange_rate__dbt_tmp - - as - --- Final base SQL model -select - id, - currency, - "DATE", - timestamp_col, - hkd_special___characters, - hkd_special___characters_1, - nzd, - usd, - "_AIRBYTE_EMITTED_AT", - "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID" -from test_normalization.dedup_exchange_rate_scd --- dedup_exchange_rate from test_normalization.airbyte_raw_dedup_exchange_rate -where "_AIRBYTE_ACTIVE_ROW" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql deleted file mode 100644 index 16dd125da9bb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql +++ /dev/null @@ -1,116 +0,0 @@ - - - create table test_normalization.dedup_exchange_rate_scd__dbt_tmp - - as - -with dbt__cte__dedup_exchange_rate_ab1__ as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value("_AIRBYTE_DATA", '$."id"') as id, - json_value("_AIRBYTE_DATA", '$."currency"') as currency, - json_value("_AIRBYTE_DATA", '$."date"') as "DATE", - json_value("_AIRBYTE_DATA", '$."timestamp_col"') as timestamp_col, - json_value("_AIRBYTE_DATA", '$."HKD@spéçiäl & characters"') as hkd_special___characters, - json_value("_AIRBYTE_DATA", '$."HKD_special___characters"') as hkd_special___characters_1, - json_value("_AIRBYTE_DATA", '$."NZD"') as nzd, - json_value("_AIRBYTE_DATA", '$."USD"') as usd, - "_AIRBYTE_EMITTED_AT" -from test_normalization.airbyte_raw_dedup_exchange_rate --- dedup_exchange_rate -), dbt__cte__dedup_exchange_rate_ab2__ as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - numeric -) as id, - cast(currency as varchar2(4000)) as currency, - cast(nullif("DATE", '') as - varchar2(4000) -) as "DATE", - cast(nullif(timestamp_col, '') as - varchar2(4000) -) as timestamp_col, - cast(hkd_special___characters as - float -) as hkd_special___characters, - cast(hkd_special___characters_1 as varchar2(4000)) as hkd_special___characters_1, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - "_AIRBYTE_EMITTED_AT" -from dbt__cte__dedup_exchange_rate_ab1__ --- dedup_exchange_rate -), dbt__cte__dedup_exchange_rate_ab3__ as ( - --- SQL model to build a hash column based on the values of this record -select - ora_hash( - - id || '~' || - - - currency || '~' || - - - "DATE" || '~' || - - - timestamp_col || '~' || - - - hkd_special___characters || '~' || - - - hkd_special___characters_1 || '~' || - - - nzd || '~' || - - - usd - - ) as "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID", - tmp.* -from dbt__cte__dedup_exchange_rate_ab2__ tmp --- dedup_exchange_rate -), dbt__cte__dedup_exchange_rate_ab4__ as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID" - order by "_AIRBYTE_EMITTED_AT" asc - ) as "_AIRBYTE_ROW_NUM", - tmp.* -from dbt__cte__dedup_exchange_rate_ab3__ tmp --- dedup_exchange_rate from test_normalization.airbyte_raw_dedup_exchange_rate -)-- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - currency, - "DATE", - timestamp_col, - hkd_special___characters, - hkd_special___characters_1, - nzd, - usd, - "DATE" as "_AIRBYTE_START_AT", - lag("DATE") over ( - partition by id, currency, cast(nzd as varchar2(4000)) - order by "DATE" asc nulls last, "DATE" desc, "_AIRBYTE_EMITTED_AT" desc - ) as "_AIRBYTE_END_AT", - case when lag("DATE") over ( - partition by id, currency, cast(nzd as varchar2(4000)) - order by "DATE" asc nulls last, "DATE" desc, "_AIRBYTE_EMITTED_AT" desc - ) is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", - "_AIRBYTE_EMITTED_AT", - "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID" -from dbt__cte__dedup_exchange_rate_ab4__ --- dedup_exchange_rate from test_normalization.airbyte_raw_dedup_exchange_rate -where "_AIRBYTE_ROW_NUM" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql deleted file mode 100644 index ee3cc52e8b43..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,95 +0,0 @@ - - - create table test_normalization.exchange_rate__dbt_tmp - - as - -with dbt__cte__exchange_rate_ab1__ as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value("_AIRBYTE_DATA", '$."id"') as id, - json_value("_AIRBYTE_DATA", '$."currency"') as currency, - json_value("_AIRBYTE_DATA", '$."date"') as "DATE", - json_value("_AIRBYTE_DATA", '$."timestamp_col"') as timestamp_col, - json_value("_AIRBYTE_DATA", '$."HKD@spéçiäl & characters"') as hkd_special___characters, - json_value("_AIRBYTE_DATA", '$."HKD_special___characters"') as hkd_special___characters_1, - json_value("_AIRBYTE_DATA", '$."NZD"') as nzd, - json_value("_AIRBYTE_DATA", '$."USD"') as usd, - "_AIRBYTE_EMITTED_AT" -from test_normalization.airbyte_raw_exchange_rate --- exchange_rate -), dbt__cte__exchange_rate_ab2__ as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - numeric -) as id, - cast(currency as varchar2(4000)) as currency, - cast(nullif("DATE", '') as - varchar2(4000) -) as "DATE", - cast(nullif(timestamp_col, '') as - varchar2(4000) -) as timestamp_col, - cast(hkd_special___characters as - float -) as hkd_special___characters, - cast(hkd_special___characters_1 as varchar2(4000)) as hkd_special___characters_1, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - "_AIRBYTE_EMITTED_AT" -from dbt__cte__exchange_rate_ab1__ --- exchange_rate -), dbt__cte__exchange_rate_ab3__ as ( - --- SQL model to build a hash column based on the values of this record -select - ora_hash( - - id || '~' || - - - currency || '~' || - - - "DATE" || '~' || - - - timestamp_col || '~' || - - - hkd_special___characters || '~' || - - - hkd_special___characters_1 || '~' || - - - nzd || '~' || - - - usd - - ) as "_AIRBYTE_EXCHANGE_RATE_HASHID", - tmp.* -from dbt__cte__exchange_rate_ab2__ tmp --- exchange_rate -)-- Final base SQL model -select - id, - currency, - "DATE", - timestamp_col, - hkd_special___characters, - hkd_special___characters_1, - nzd, - usd, - "_AIRBYTE_EMITTED_AT", - "_AIRBYTE_EXCHANGE_RATE_HASHID" -from dbt__cte__exchange_rate_ab3__ --- exchange_rate from test_normalization.airbyte_raw_exchange_rate \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql deleted file mode 100644 index 48197614eaaa..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create table test_normalization.pos_dedup_cdcx__dbt_tmp - - as - --- Final base SQL model -select - id, - name, - "_AB_CDC_LSN", - "_AB_CDC_UPDATED_AT", - "_AB_CDC_DELETED_AT", - "_AB_CDC_LOG_POS", - "_AIRBYTE_EMITTED_AT", - "_AIRBYTE_POS_DEDUP_CDCX_HASHID" -from test_normalization.pos_dedup_cdcx_scd --- pos_dedup_cdcx from test_normalization.airbyte_raw_pos_dedup_cdcx -where "_AIRBYTE_ACTIVE_ROW" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index be9d106c3db0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,102 +0,0 @@ - - - create table test_normalization.pos_dedup_cdcx_scd__dbt_tmp - - as - -with dbt__cte__pos_dedup_cdcx_ab1__ as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value("_AIRBYTE_DATA", '$."id"') as id, - json_value("_AIRBYTE_DATA", '$."name"') as name, - json_value("_AIRBYTE_DATA", '$."_ab_cdc_lsn"') as "_AB_CDC_LSN", - json_value("_AIRBYTE_DATA", '$."_ab_cdc_updated_at"') as "_AB_CDC_UPDATED_AT", - json_value("_AIRBYTE_DATA", '$."_ab_cdc_deleted_at"') as "_AB_CDC_DELETED_AT", - json_value("_AIRBYTE_DATA", '$."_ab_cdc_log_pos"') as "_AB_CDC_LOG_POS", - "_AIRBYTE_EMITTED_AT" -from test_normalization.airbyte_raw_pos_dedup_cdcx --- pos_dedup_cdcx -), dbt__cte__pos_dedup_cdcx_ab2__ as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - numeric -) as id, - cast(name as varchar2(4000)) as name, - cast("_AB_CDC_LSN" as - float -) as "_AB_CDC_LSN", - cast("_AB_CDC_UPDATED_AT" as - float -) as "_AB_CDC_UPDATED_AT", - cast("_AB_CDC_DELETED_AT" as - float -) as "_AB_CDC_DELETED_AT", - cast("_AB_CDC_LOG_POS" as - float -) as "_AB_CDC_LOG_POS", - "_AIRBYTE_EMITTED_AT" -from dbt__cte__pos_dedup_cdcx_ab1__ --- pos_dedup_cdcx -), dbt__cte__pos_dedup_cdcx_ab3__ as ( - --- SQL model to build a hash column based on the values of this record -select - ora_hash( - - id || '~' || - - - name || '~' || - - - "_AB_CDC_LSN" || '~' || - - - "_AB_CDC_UPDATED_AT" || '~' || - - - "_AB_CDC_DELETED_AT" || '~' || - - - "_AB_CDC_LOG_POS" - - ) as "_AIRBYTE_POS_DEDUP_CDCX_HASHID", - tmp.* -from dbt__cte__pos_dedup_cdcx_ab2__ tmp --- pos_dedup_cdcx -), dbt__cte__pos_dedup_cdcx_ab4__ as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by "_AIRBYTE_POS_DEDUP_CDCX_HASHID" - order by "_AIRBYTE_EMITTED_AT" asc - ) as "_AIRBYTE_ROW_NUM", - tmp.* -from dbt__cte__pos_dedup_cdcx_ab3__ tmp --- pos_dedup_cdcx from test_normalization.airbyte_raw_pos_dedup_cdcx -)-- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - "_AB_CDC_LSN", - "_AB_CDC_UPDATED_AT", - "_AB_CDC_DELETED_AT", - "_AB_CDC_LOG_POS", - "_AIRBYTE_EMITTED_AT" as "_AIRBYTE_START_AT", - lag("_AIRBYTE_EMITTED_AT") over ( - partition by id - order by "_AIRBYTE_EMITTED_AT" asc nulls last, "_AIRBYTE_EMITTED_AT" desc, "_AIRBYTE_EMITTED_AT" desc - ) as "_AIRBYTE_END_AT", - case when lag("_AIRBYTE_EMITTED_AT") over ( - partition by id - order by "_AIRBYTE_EMITTED_AT" asc nulls last, "_AIRBYTE_EMITTED_AT" desc, "_AIRBYTE_EMITTED_AT" desc, "_AB_CDC_UPDATED_AT" desc, "_AB_CDC_LOG_POS" desc - ) is null and "_AB_CDC_DELETED_AT" is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", - "_AIRBYTE_EMITTED_AT", - "_AIRBYTE_POS_DEDUP_CDCX_HASHID" -from dbt__cte__pos_dedup_cdcx_ab4__ --- pos_dedup_cdcx from test_normalization.airbyte_raw_pos_dedup_cdcx -where "_AIRBYTE_ROW_NUM" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..bd51b214ebc6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,93 @@ + + + create table test_normalization.dedup_cdc_excluded_scd__dbt_tmp + + as + +with + +input_data as ( + select * + from test_normalization.dedup_cdc_excluded_ab3 + -- dedup_cdc_excluded from test_normalization.airbyte_raw_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + ora_hash( + + id + + ) as "_AIRBYTE_UNIQUE_KEY", + id, + name, + "_AB_CDC_LSN", + "_AB_CDC_UPDATED_AT", + "_AB_CDC_DELETED_AT", + "_AIRBYTE_EMITTED_AT" as "_AIRBYTE_START_AT", + lag("_AIRBYTE_EMITTED_AT") over ( + partition by id + order by + "_AIRBYTE_EMITTED_AT" asc nulls last, + "_AIRBYTE_EMITTED_AT" desc, + "_AIRBYTE_EMITTED_AT" desc, "_AB_CDC_UPDATED_AT" desc + ) as "_AIRBYTE_END_AT", + case when lag("_AIRBYTE_EMITTED_AT") over ( + partition by id + order by + "_AIRBYTE_EMITTED_AT" asc nulls last, + "_AIRBYTE_EMITTED_AT" desc, + "_AIRBYTE_EMITTED_AT" desc, "_AB_CDC_UPDATED_AT" desc + ) is null and "_AB_CDC_DELETED_AT" is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_START_AT", "_AIRBYTE_EMITTED_AT", cast("_AB_CDC_DELETED_AT" as varchar2(4000)), cast("_AB_CDC_UPDATED_AT" as varchar2(4000)) + order by "_AIRBYTE_AB_ID" + ) as "_AIRBYTE_ROW_NUM", + ora_hash( + + "_AIRBYTE_UNIQUE_KEY" || '~' || + + + "_AIRBYTE_START_AT" || '~' || + + + "_AIRBYTE_EMITTED_AT" || '~' || + + + "_AB_CDC_DELETED_AT" || '~' || + + + "_AB_CDC_UPDATED_AT" + + ) as "_AIRBYTE_UNIQUE_KEY_SCD", + scd_data.* + from scd_data +) +select + "_AIRBYTE_UNIQUE_KEY", + "_AIRBYTE_UNIQUE_KEY_SCD", + id, + name, + "_AB_CDC_LSN", + "_AB_CDC_UPDATED_AT", + "_AB_CDC_DELETED_AT", + "_AIRBYTE_START_AT", + "_AIRBYTE_END_AT", + "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" +from dedup_data where "_AIRBYTE_ROW_NUM" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..e6c91980fea5 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,99 @@ + + + create table test_normalization.dedup_exchange_rate_scd__dbt_tmp + + as + +with + +input_data as ( + select * + from test_normalization.dedup_exchange_rate_ab3 + -- dedup_exchange_rate from test_normalization.airbyte_raw_dedup_exchange_rate +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + ora_hash( + + id || '~' || + + + currency || '~' || + + + nzd + + ) as "_AIRBYTE_UNIQUE_KEY", + id, + currency, + "DATE", + timestamp_col, + hkd_special___characters, + hkd_special___characters_1, + nzd, + usd, + "DATE" as "_AIRBYTE_START_AT", + lag("DATE") over ( + partition by id, currency, cast(nzd as varchar2(4000)) + order by + "DATE" asc nulls last, + "DATE" desc, + "_AIRBYTE_EMITTED_AT" desc + ) as "_AIRBYTE_END_AT", + case when lag("DATE") over ( + partition by id, currency, cast(nzd as varchar2(4000)) + order by + "DATE" asc nulls last, + "DATE" desc, + "_AIRBYTE_EMITTED_AT" desc + ) is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID" + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_START_AT", "_AIRBYTE_EMITTED_AT" + order by "_AIRBYTE_AB_ID" + ) as "_AIRBYTE_ROW_NUM", + ora_hash( + + "_AIRBYTE_UNIQUE_KEY" || '~' || + + + "_AIRBYTE_START_AT" || '~' || + + + "_AIRBYTE_EMITTED_AT" + + ) as "_AIRBYTE_UNIQUE_KEY_SCD", + scd_data.* + from scd_data +) +select + "_AIRBYTE_UNIQUE_KEY", + "_AIRBYTE_UNIQUE_KEY_SCD", + id, + currency, + "DATE", + timestamp_col, + hkd_special___characters, + hkd_special___characters_1, + nzd, + usd, + "_AIRBYTE_START_AT", + "_AIRBYTE_END_AT", + "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID" +from dedup_data where "_AIRBYTE_ROW_NUM" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..c8d9914d17d2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,79 @@ + + + create table test_normalization.renamed_dedup_cdc_excluded_scd__dbt_tmp + + as + +with + +input_data as ( + select * + from test_normalization.renamed_dedup_cdc_excluded_ab3 + -- renamed_dedup_cdc_excluded from test_normalization.airbyte_raw_renamed_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + ora_hash( + + id + + ) as "_AIRBYTE_UNIQUE_KEY", + id, + "_AIRBYTE_EMITTED_AT" as "_AIRBYTE_START_AT", + lag("_AIRBYTE_EMITTED_AT") over ( + partition by id + order by + "_AIRBYTE_EMITTED_AT" asc nulls last, + "_AIRBYTE_EMITTED_AT" desc, + "_AIRBYTE_EMITTED_AT" desc + ) as "_AIRBYTE_END_AT", + case when lag("_AIRBYTE_EMITTED_AT") over ( + partition by id + order by + "_AIRBYTE_EMITTED_AT" asc nulls last, + "_AIRBYTE_EMITTED_AT" desc, + "_AIRBYTE_EMITTED_AT" desc + ) is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_START_AT", "_AIRBYTE_EMITTED_AT" + order by "_AIRBYTE_AB_ID" + ) as "_AIRBYTE_ROW_NUM", + ora_hash( + + "_AIRBYTE_UNIQUE_KEY" || '~' || + + + "_AIRBYTE_START_AT" || '~' || + + + "_AIRBYTE_EMITTED_AT" + + ) as "_AIRBYTE_UNIQUE_KEY_SCD", + scd_data.* + from scd_data +) +select + "_AIRBYTE_UNIQUE_KEY", + "_AIRBYTE_UNIQUE_KEY_SCD", + id, + "_AIRBYTE_START_AT", + "_AIRBYTE_END_AT", + "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" +from dedup_data where "_AIRBYTE_ROW_NUM" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql new file mode 100644 index 000000000000..bf4c8a11c07b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -0,0 +1,24 @@ + + + create table test_normalization.dedup_cdc_excluded__dbt_tmp + + as + +-- Final base SQL model +select + "_AIRBYTE_UNIQUE_KEY", + id, + name, + "_AB_CDC_LSN", + "_AB_CDC_UPDATED_AT", + "_AB_CDC_DELETED_AT", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" +from test_normalization.dedup_cdc_excluded_scd +-- dedup_cdc_excluded from test_normalization.airbyte_raw_dedup_cdc_excluded +where 1 = 1 +and "_AIRBYTE_ACTIVE_ROW" = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..f67f7384cf17 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,27 @@ + + + create table test_normalization.dedup_exchange_rate__dbt_tmp + + as + +-- Final base SQL model +select + "_AIRBYTE_UNIQUE_KEY", + id, + currency, + "DATE", + timestamp_col, + hkd_special___characters, + hkd_special___characters_1, + nzd, + usd, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID" +from test_normalization.dedup_exchange_rate_scd +-- dedup_exchange_rate from test_normalization.airbyte_raw_dedup_exchange_rate +where 1 = 1 +and "_AIRBYTE_ACTIVE_ROW" = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..9d77656b65a9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -0,0 +1,114 @@ + + + create table test_normalization.exchange_rate__dbt_tmp + + as + +with dbt__cte__exchange_rate_ab1__ as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value("_AIRBYTE_DATA", '$."id"') as id, + json_value("_AIRBYTE_DATA", '$."currency"') as currency, + json_value("_AIRBYTE_DATA", '$."date"') as "DATE", + json_value("_AIRBYTE_DATA", '$."timestamp_col"') as timestamp_col, + json_value("_AIRBYTE_DATA", '$."HKD@spéçiäl & characters"') as hkd_special___characters, + json_value("_AIRBYTE_DATA", '$."HKD_special___characters"') as hkd_special___characters_1, + json_value("_AIRBYTE_DATA", '$."NZD"') as nzd, + json_value("_AIRBYTE_DATA", '$."USD"') as usd, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from test_normalization.airbyte_raw_exchange_rate +-- exchange_rate +where 1 = 1 + +), dbt__cte__exchange_rate_ab2__ as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + numeric +) as id, + cast(currency as varchar2(4000)) as currency, + cast(nullif("DATE", '') as + varchar2(4000) +) as "DATE", + cast(nullif(timestamp_col, '') as + varchar2(4000) +) as timestamp_col, + cast(hkd_special___characters as + float +) as hkd_special___characters, + cast(hkd_special___characters_1 as varchar2(4000)) as hkd_special___characters_1, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from dbt__cte__exchange_rate_ab1__ +-- exchange_rate +where 1 = 1 + +), dbt__cte__exchange_rate_ab3__ as ( + +-- SQL model to build a hash column based on the values of this record +select + ora_hash( + + id || '~' || + + + currency || '~' || + + + "DATE" || '~' || + + + timestamp_col || '~' || + + + hkd_special___characters || '~' || + + + hkd_special___characters_1 || '~' || + + + nzd || '~' || + + + usd + + ) as "_AIRBYTE_EXCHANGE_RATE_HASHID", + tmp.* +from dbt__cte__exchange_rate_ab2__ tmp +-- exchange_rate +where 1 = 1 + +)-- Final base SQL model +select + id, + currency, + "DATE", + timestamp_col, + hkd_special___characters, + hkd_special___characters_1, + nzd, + usd, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_EXCHANGE_RATE_HASHID" +from dbt__cte__exchange_rate_ab3__ +-- exchange_rate from test_normalization.airbyte_raw_exchange_rate +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..feac3368a6ca --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,20 @@ + + + create table test_normalization.renamed_dedup_cdc_excluded__dbt_tmp + + as + +-- Final base SQL model +select + "_AIRBYTE_UNIQUE_KEY", + id, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" +from test_normalization.renamed_dedup_cdc_excluded_scd +-- renamed_dedup_cdc_excluded from test_normalization.airbyte_raw_renamed_dedup_cdc_excluded +where 1 = 1 +and "_AIRBYTE_ACTIVE_ROW" = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..c184fd31e2d3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,98 @@ + + + create table test_normalization.pos_dedup_cdcx_scd__dbt_tmp + + as + +with + +input_data as ( + select * + from test_normalization.pos_dedup_cdcx_ab3 + -- pos_dedup_cdcx from test_normalization.airbyte_raw_pos_dedup_cdcx +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + ora_hash( + + id + + ) as "_AIRBYTE_UNIQUE_KEY", + id, + name, + "_AB_CDC_LSN", + "_AB_CDC_UPDATED_AT", + "_AB_CDC_DELETED_AT", + "_AB_CDC_LOG_POS", + "_AIRBYTE_EMITTED_AT" as "_AIRBYTE_START_AT", + lag("_AIRBYTE_EMITTED_AT") over ( + partition by id + order by + "_AIRBYTE_EMITTED_AT" asc nulls last, + "_AIRBYTE_EMITTED_AT" desc, + "_AIRBYTE_EMITTED_AT" desc, "_AB_CDC_UPDATED_AT" desc, "_AB_CDC_LOG_POS" desc + ) as "_AIRBYTE_END_AT", + case when lag("_AIRBYTE_EMITTED_AT") over ( + partition by id + order by + "_AIRBYTE_EMITTED_AT" asc nulls last, + "_AIRBYTE_EMITTED_AT" desc, + "_AIRBYTE_EMITTED_AT" desc, "_AB_CDC_UPDATED_AT" desc, "_AB_CDC_LOG_POS" desc + ) is null and "_AB_CDC_DELETED_AT" is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + "_AIRBYTE_POS_DEDUP_CDCX_HASHID" + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_START_AT", "_AIRBYTE_EMITTED_AT", cast("_AB_CDC_DELETED_AT" as varchar2(4000)), cast("_AB_CDC_UPDATED_AT" as varchar2(4000)), cast("_AB_CDC_LOG_POS" as varchar2(4000)) + order by "_AIRBYTE_AB_ID" + ) as "_AIRBYTE_ROW_NUM", + ora_hash( + + "_AIRBYTE_UNIQUE_KEY" || '~' || + + + "_AIRBYTE_START_AT" || '~' || + + + "_AIRBYTE_EMITTED_AT" || '~' || + + + "_AB_CDC_DELETED_AT" || '~' || + + + "_AB_CDC_UPDATED_AT" || '~' || + + + "_AB_CDC_LOG_POS" + + ) as "_AIRBYTE_UNIQUE_KEY_SCD", + scd_data.* + from scd_data +) +select + "_AIRBYTE_UNIQUE_KEY", + "_AIRBYTE_UNIQUE_KEY_SCD", + id, + name, + "_AB_CDC_LSN", + "_AB_CDC_UPDATED_AT", + "_AB_CDC_DELETED_AT", + "_AB_CDC_LOG_POS", + "_AIRBYTE_START_AT", + "_AIRBYTE_END_AT", + "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_POS_DEDUP_CDCX_HASHID" +from dedup_data where "_AIRBYTE_ROW_NUM" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql new file mode 100644 index 000000000000..d3fd87ee5576 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -0,0 +1,25 @@ + + + create table test_normalization.pos_dedup_cdcx__dbt_tmp + + as + +-- Final base SQL model +select + "_AIRBYTE_UNIQUE_KEY", + id, + name, + "_AB_CDC_LSN", + "_AB_CDC_UPDATED_AT", + "_AB_CDC_DELETED_AT", + "_AB_CDC_LOG_POS", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_POS_DEDUP_CDCX_HASHID" +from test_normalization.pos_dedup_cdcx_scd +-- pos_dedup_cdcx from test_normalization.airbyte_raw_pos_dedup_cdcx +where 1 = 1 +and "_AIRBYTE_ACTIVE_ROW" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..30135e3c86ec --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,72 @@ + + create view test_normalization.dedup_cdc_excluded_ab3__dbt_tmp as + +with dbt__cte__dedup_cdc_excluded_ab1__ as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value("_AIRBYTE_DATA", '$."id"') as id, + json_value("_AIRBYTE_DATA", '$."name"') as name, + json_value("_AIRBYTE_DATA", '$."_ab_cdc_lsn"') as "_AB_CDC_LSN", + json_value("_AIRBYTE_DATA", '$."_ab_cdc_updated_at"') as "_AB_CDC_UPDATED_AT", + json_value("_AIRBYTE_DATA", '$."_ab_cdc_deleted_at"') as "_AB_CDC_DELETED_AT", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from test_normalization.airbyte_raw_dedup_cdc_excluded +-- dedup_cdc_excluded +where 1 = 1 + +), dbt__cte__dedup_cdc_excluded_ab2__ as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + numeric +) as id, + cast(name as varchar2(4000)) as name, + cast("_AB_CDC_LSN" as + float +) as "_AB_CDC_LSN", + cast("_AB_CDC_UPDATED_AT" as + float +) as "_AB_CDC_UPDATED_AT", + cast("_AB_CDC_DELETED_AT" as + float +) as "_AB_CDC_DELETED_AT", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from dbt__cte__dedup_cdc_excluded_ab1__ +-- dedup_cdc_excluded +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + ora_hash( + + id || '~' || + + + name || '~' || + + + "_AB_CDC_LSN" || '~' || + + + "_AB_CDC_UPDATED_AT" || '~' || + + + "_AB_CDC_DELETED_AT" + + ) as "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID", + tmp.* +from dbt__cte__dedup_cdc_excluded_ab2__ tmp +-- dedup_cdc_excluded +where 1 = 1 + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..527196251304 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,91 @@ + + create view test_normalization.dedup_exchange_rate_ab3__dbt_tmp as + +with dbt__cte__dedup_exchange_rate_ab1__ as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value("_AIRBYTE_DATA", '$."id"') as id, + json_value("_AIRBYTE_DATA", '$."currency"') as currency, + json_value("_AIRBYTE_DATA", '$."date"') as "DATE", + json_value("_AIRBYTE_DATA", '$."timestamp_col"') as timestamp_col, + json_value("_AIRBYTE_DATA", '$."HKD@spéçiäl & characters"') as hkd_special___characters, + json_value("_AIRBYTE_DATA", '$."HKD_special___characters"') as hkd_special___characters_1, + json_value("_AIRBYTE_DATA", '$."NZD"') as nzd, + json_value("_AIRBYTE_DATA", '$."USD"') as usd, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from test_normalization.airbyte_raw_dedup_exchange_rate +-- dedup_exchange_rate +where 1 = 1 + +), dbt__cte__dedup_exchange_rate_ab2__ as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + numeric +) as id, + cast(currency as varchar2(4000)) as currency, + cast(nullif("DATE", '') as + varchar2(4000) +) as "DATE", + cast(nullif(timestamp_col, '') as + varchar2(4000) +) as timestamp_col, + cast(hkd_special___characters as + float +) as hkd_special___characters, + cast(hkd_special___characters_1 as varchar2(4000)) as hkd_special___characters_1, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from dbt__cte__dedup_exchange_rate_ab1__ +-- dedup_exchange_rate +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + ora_hash( + + id || '~' || + + + currency || '~' || + + + "DATE" || '~' || + + + timestamp_col || '~' || + + + hkd_special___characters || '~' || + + + hkd_special___characters_1 || '~' || + + + nzd || '~' || + + + usd + + ) as "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID", + tmp.* +from dbt__cte__dedup_exchange_rate_ab2__ tmp +-- dedup_exchange_rate +where 1 = 1 + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql new file mode 100644 index 000000000000..e1a0af83a632 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -0,0 +1,76 @@ + + create view test_normalization.pos_dedup_cdcx_ab3__dbt_tmp as + +with dbt__cte__pos_dedup_cdcx_ab1__ as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value("_AIRBYTE_DATA", '$."id"') as id, + json_value("_AIRBYTE_DATA", '$."name"') as name, + json_value("_AIRBYTE_DATA", '$."_ab_cdc_lsn"') as "_AB_CDC_LSN", + json_value("_AIRBYTE_DATA", '$."_ab_cdc_updated_at"') as "_AB_CDC_UPDATED_AT", + json_value("_AIRBYTE_DATA", '$."_ab_cdc_deleted_at"') as "_AB_CDC_DELETED_AT", + json_value("_AIRBYTE_DATA", '$."_ab_cdc_log_pos"') as "_AB_CDC_LOG_POS", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from test_normalization.airbyte_raw_pos_dedup_cdcx +-- pos_dedup_cdcx +where 1 = 1 +), dbt__cte__pos_dedup_cdcx_ab2__ as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + numeric +) as id, + cast(name as varchar2(4000)) as name, + cast("_AB_CDC_LSN" as + float +) as "_AB_CDC_LSN", + cast("_AB_CDC_UPDATED_AT" as + float +) as "_AB_CDC_UPDATED_AT", + cast("_AB_CDC_DELETED_AT" as + float +) as "_AB_CDC_DELETED_AT", + cast("_AB_CDC_LOG_POS" as + float +) as "_AB_CDC_LOG_POS", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from dbt__cte__pos_dedup_cdcx_ab1__ +-- pos_dedup_cdcx +where 1 = 1 +)-- SQL model to build a hash column based on the values of this record +select + ora_hash( + + id || '~' || + + + name || '~' || + + + "_AB_CDC_LSN" || '~' || + + + "_AB_CDC_UPDATED_AT" || '~' || + + + "_AB_CDC_DELETED_AT" || '~' || + + + "_AB_CDC_LOG_POS" + + ) as "_AIRBYTE_POS_DEDUP_CDCX_HASHID", + tmp.* +from dbt__cte__pos_dedup_cdcx_ab2__ tmp +-- pos_dedup_cdcx +where 1 = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..6ad67cd02a90 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,46 @@ + + create view test_normalization.renamed_dedup_cdc_excluded_ab3__dbt_tmp as + +with dbt__cte__renamed_dedup_cdc_excluded_ab1__ as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value("_AIRBYTE_DATA", '$."id"') as id, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from test_normalization.airbyte_raw_renamed_dedup_cdc_excluded +-- renamed_dedup_cdc_excluded +where 1 = 1 + +), dbt__cte__renamed_dedup_cdc_excluded_ab2__ as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + numeric +) as id, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from dbt__cte__renamed_dedup_cdc_excluded_ab1__ +-- renamed_dedup_cdc_excluded +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + ora_hash( + + id + + ) as "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID", + tmp.* +from dbt__cte__renamed_dedup_cdc_excluded_ab2__ tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql deleted file mode 100644 index fa2cc2ec8b0b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql +++ /dev/null @@ -1,43 +0,0 @@ -with table_row_counts as ( - select distinct count(*) as row_count, 10 as expected_count - from {{ source('test_normalization', 'airbyte_raw_exchange_rate') }} -union all - select distinct count(*) as row_count, 10 as expected_count - from {{ ref('exchange_rate') }} - -union all - - select distinct count(*) as row_count, 10 as expected_count - from {{ source('test_normalization', 'airbyte_raw_dedup_exchange_rate') }} -union all - select distinct count(*) as row_count, 10 as expected_count - from {{ ref('dedup_exchange_rate_scd') }} -union all - select distinct count(*) as row_count, 5 as expected_count - from {{ ref('dedup_exchange_rate') }} - -union all - - select distinct count(*) as row_count, 8 as expected_count - from {{ source('test_normalization', 'airbyte_raw_dedup_cdc_excluded') }} -union all - select distinct count(*) as row_count, 8 as expected_count - from {{ ref('dedup_cdc_excluded_scd') }} -union all - select distinct count(*) as row_count, 4 as expected_count - from {{ ref('dedup_cdc_excluded') }} - -union all - - select distinct count(*) as row_count, 8 as expected_count - from {{ source('test_normalization', 'airbyte_raw_pos_dedup_cdcx') }} -union all - select distinct count(*) as row_count, 8 as expected_count - from {{ ref('pos_dedup_cdcx_scd') }} -union all - select distinct count(*) as row_count, 3 as expected_count - from {{ ref('pos_dedup_cdcx') }} -) -select * -from table_row_counts -where row_count != expected_count diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql index b96dabd43418..2f3b6d1b7104 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -1,13 +1,20 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['id'], ['id']) }} as id, {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['name'], ['name']) }} as name, - {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['column`_\'with"_quotes'], ['column___with__quotes']) }} as column___with__quotes, {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['_ab_cdc_lsn'], ['_ab_cdc_lsn']) }} as {{ quote('_AB_CDC_LSN') }}, {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as {{ quote('_AB_CDC_UPDATED_AT') }}, {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as {{ quote('_AB_CDC_DELETED_AT') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }} + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }} from {{ source('test_normalization', 'airbyte_raw_dedup_cdc_excluded') }} -- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql index 00175dd65806..9f7cb197462d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -1,13 +1,20 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, cast(name as {{ dbt_utils.type_string() }}) as name, - cast(column___with__quotes as {{ dbt_utils.type_string() }}) as column___with__quotes, cast({{ quote('_AB_CDC_LSN') }} as {{ dbt_utils.type_float() }}) as {{ quote('_AB_CDC_LSN') }}, cast({{ quote('_AB_CDC_UPDATED_AT') }} as {{ dbt_utils.type_float() }}) as {{ quote('_AB_CDC_UPDATED_AT') }}, cast({{ quote('_AB_CDC_DELETED_AT') }} as {{ dbt_utils.type_float() }}) as {{ quote('_AB_CDC_DELETED_AT') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }} + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }} from {{ ref('dedup_cdc_excluded_ab1') }} -- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index da67fdfcb24e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'name', - 'column___with__quotes', - quote('_AB_CDC_LSN'), - quote('_AB_CDC_UPDATED_AT'), - quote('_AB_CDC_DELETED_AT'), - ]) }} as {{ quote('_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID') }}, - tmp.* -from {{ ref('dedup_cdc_excluded_ab2') }} tmp --- dedup_cdc_excluded - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql deleted file mode 100644 index 7991b32d4157..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by {{ quote('_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID') }} - order by {{ quote('_AIRBYTE_EMITTED_AT') }} asc - ) as {{ quote('_AIRBYTE_ROW_NUM') }}, - tmp.* -from {{ ref('dedup_cdc_excluded_ab3') }} tmp --- dedup_cdc_excluded from {{ source('test_normalization', 'airbyte_raw_dedup_cdc_excluded') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql index e71a865c767e..a0dda141cd02 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -1,4 +1,8 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['id'], ['id']) }} as id, @@ -9,7 +13,11 @@ select {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['HKD_special___characters'], ['HKD_special___characters']) }} as hkd_special___characters_1, {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['NZD'], ['NZD']) }} as nzd, {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['USD'], ['USD']) }} as usd, - {{ quote('_AIRBYTE_EMITTED_AT') }} + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }} from {{ source('test_normalization', 'airbyte_raw_dedup_exchange_rate') }} -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index e79c6d54210d..fadc49c40be3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -1,4 +1,8 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -9,7 +13,11 @@ select cast(hkd_special___characters_1 as {{ dbt_utils.type_string() }}) as hkd_special___characters_1, cast(nzd as {{ dbt_utils.type_float() }}) as nzd, cast(usd as {{ dbt_utils.type_float() }}) as usd, - {{ quote('_AIRBYTE_EMITTED_AT') }} + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }} from {{ ref('dedup_exchange_rate_ab1') }} -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql deleted file mode 100644 index 32f16e62fef2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'currency', - quote('DATE'), - 'timestamp_col', - 'hkd_special___characters', - 'hkd_special___characters_1', - 'nzd', - 'usd', - ]) }} as {{ quote('_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID') }}, - tmp.* -from {{ ref('dedup_exchange_rate_ab2') }} tmp --- dedup_exchange_rate - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql deleted file mode 100644 index 28d9f65311da..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by {{ quote('_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID') }} - order by {{ quote('_AIRBYTE_EMITTED_AT') }} asc - ) as {{ quote('_AIRBYTE_ROW_NUM') }}, - tmp.* -from {{ ref('dedup_exchange_rate_ab3') }} tmp --- dedup_exchange_rate from {{ source('test_normalization', 'airbyte_raw_dedup_exchange_rate') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql index 130eb4e0e636..bcc1db00d8e3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -1,4 +1,8 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['id'], ['id']) }} as id, @@ -9,7 +13,11 @@ select {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['HKD_special___characters'], ['HKD_special___characters']) }} as hkd_special___characters_1, {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['NZD'], ['NZD']) }} as nzd, {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['USD'], ['USD']) }} as usd, - {{ quote('_AIRBYTE_EMITTED_AT') }} + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }} from {{ source('test_normalization', 'airbyte_raw_exchange_rate') }} -- exchange_rate +where 1 = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql index 63adb62a688e..e7f5e3353cfd 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -1,4 +1,8 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -9,7 +13,11 @@ select cast(hkd_special___characters_1 as {{ dbt_utils.type_string() }}) as hkd_special___characters_1, cast(nzd as {{ dbt_utils.type_float() }}) as nzd, cast(usd as {{ dbt_utils.type_float() }}) as usd, - {{ quote('_AIRBYTE_EMITTED_AT') }} + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }} from {{ ref('exchange_rate_ab1') }} -- exchange_rate +where 1 = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql index 2cf009488395..21fb107c1c44 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -14,4 +18,6 @@ select tmp.* from {{ ref('exchange_rate_ab2') }} tmp -- exchange_rate +where 1 = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql index 3cc4f3f73faf..9cb45ceefec4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -1,4 +1,8 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['id'], ['id']) }} as id, @@ -7,7 +11,10 @@ select {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as {{ quote('_AB_CDC_UPDATED_AT') }}, {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as {{ quote('_AB_CDC_DELETED_AT') }}, {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['_ab_cdc_log_pos'], ['_ab_cdc_log_pos']) }} as {{ quote('_AB_CDC_LOG_POS') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }} + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }} from {{ source('test_normalization', 'airbyte_raw_pos_dedup_cdcx') }} -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql index 344413a6b470..de537cd43138 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -1,4 +1,8 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -7,7 +11,10 @@ select cast({{ quote('_AB_CDC_UPDATED_AT') }} as {{ dbt_utils.type_float() }}) as {{ quote('_AB_CDC_UPDATED_AT') }}, cast({{ quote('_AB_CDC_DELETED_AT') }} as {{ dbt_utils.type_float() }}) as {{ quote('_AB_CDC_DELETED_AT') }}, cast({{ quote('_AB_CDC_LOG_POS') }} as {{ dbt_utils.type_float() }}) as {{ quote('_AB_CDC_LOG_POS') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }} + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }} from {{ ref('pos_dedup_cdcx_ab1') }} -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index bbe9333a3383..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'name', - quote('_AB_CDC_LSN'), - quote('_AB_CDC_UPDATED_AT'), - quote('_AB_CDC_DELETED_AT'), - quote('_AB_CDC_LOG_POS'), - ]) }} as {{ quote('_AIRBYTE_POS_DEDUP_CDCX_HASHID') }}, - tmp.* -from {{ ref('pos_dedup_cdcx_ab2') }} tmp --- pos_dedup_cdcx - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql deleted file mode 100644 index 8734f4c27df5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by {{ quote('_AIRBYTE_POS_DEDUP_CDCX_HASHID') }} - order by {{ quote('_AIRBYTE_EMITTED_AT') }} asc - ) as {{ quote('_AIRBYTE_ROW_NUM') }}, - tmp.* -from {{ ref('pos_dedup_cdcx_ab3') }} tmp --- pos_dedup_cdcx from {{ source('test_normalization', 'airbyte_raw_pos_dedup_cdcx') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..e7fe7d6f462c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,16 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['id'], ['id']) }} as id, + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }} +from {{ source('test_normalization', 'airbyte_raw_renamed_dedup_cdc_excluded') }} +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..fc3191e47f30 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,16 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as {{ dbt_utils.type_bigint() }}) as id, + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }} +from {{ ref('renamed_dedup_cdc_excluded_ab1') }} +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..3f81f2997f7c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,111 @@ +{{ config( + unique_key = "{{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('dedup_cdc_excluded_ab3') }} + -- dedup_cdc_excluded from {{ source('test_normalization', 'airbyte_raw_dedup_cdc_excluded') }} + where 1 = 1 + {{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} +), +new_data_ids as ( + -- build a subset of {{ quote('_AIRBYTE_UNIQUE_KEY') }} from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY') }} + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data.{{ quote('_AIRBYTE_UNIQUE_KEY') }} = new_data_ids.{{ quote('_AIRBYTE_UNIQUE_KEY') }} + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 + where {{ quote('_AIRBYTE_ACTIVE_ROW') }} = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('dedup_cdc_excluded_ab3') }} + -- dedup_cdc_excluded from {{ source('test_normalization', 'airbyte_raw_dedup_cdc_excluded') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY') }}, + id, + name, + {{ quote('_AB_CDC_LSN') }}, + {{ quote('_AB_CDC_UPDATED_AT') }}, + {{ quote('_AB_CDC_DELETED_AT') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }} as {{ quote('_AIRBYTE_START_AT') }}, + lag({{ quote('_AIRBYTE_EMITTED_AT') }}) over ( + partition by id + order by + {{ quote('_AIRBYTE_EMITTED_AT') }} asc nulls last, + {{ quote('_AIRBYTE_EMITTED_AT') }} desc, + {{ quote('_AIRBYTE_EMITTED_AT') }} desc, {{ quote('_AB_CDC_UPDATED_AT') }} desc + ) as {{ quote('_AIRBYTE_END_AT') }}, + case when lag({{ quote('_AIRBYTE_EMITTED_AT') }}) over ( + partition by id + order by + {{ quote('_AIRBYTE_EMITTED_AT') }} asc nulls last, + {{ quote('_AIRBYTE_EMITTED_AT') }} desc, + {{ quote('_AIRBYTE_EMITTED_AT') }} desc, {{ quote('_AB_CDC_UPDATED_AT') }} desc + ) is null and {{ quote('_AB_CDC_DELETED_AT') }} is null then 1 else 0 end as {{ quote('_AIRBYTE_ACTIVE_ROW') }}, + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ quote('_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID') }} + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by {{ quote('_AIRBYTE_UNIQUE_KEY') }}, {{ quote('_AIRBYTE_START_AT') }}, {{ quote('_AIRBYTE_EMITTED_AT') }}, cast({{ quote('_AB_CDC_DELETED_AT') }} as {{ dbt_utils.type_string() }}), cast({{ quote('_AB_CDC_UPDATED_AT') }} as {{ dbt_utils.type_string() }}) + order by {{ quote('_AIRBYTE_AB_ID') }} + ) as {{ quote('_AIRBYTE_ROW_NUM') }}, + {{ dbt_utils.surrogate_key([ + quote('_AIRBYTE_UNIQUE_KEY'), + quote('_AIRBYTE_START_AT'), + quote('_AIRBYTE_EMITTED_AT'), quote('_AB_CDC_DELETED_AT'), quote('_AB_CDC_UPDATED_AT') + ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}, + scd_data.* + from scd_data +) +select + {{ quote('_AIRBYTE_UNIQUE_KEY') }}, + {{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}, + id, + name, + {{ quote('_AB_CDC_LSN') }}, + {{ quote('_AB_CDC_UPDATED_AT') }}, + {{ quote('_AB_CDC_DELETED_AT') }}, + {{ quote('_AIRBYTE_START_AT') }}, + {{ quote('_AIRBYTE_END_AT') }}, + {{ quote('_AIRBYTE_ACTIVE_ROW') }}, + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }}, + {{ quote('_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID') }} +from dedup_data where {{ quote('_AIRBYTE_ROW_NUM') }} = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..6ad3983885b4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,121 @@ +{{ config( + unique_key = "{{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('dedup_exchange_rate_ab3') }} + -- dedup_exchange_rate from {{ source('test_normalization', 'airbyte_raw_dedup_exchange_rate') }} + where 1 = 1 + {{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} +), +new_data_ids as ( + -- build a subset of {{ quote('_AIRBYTE_UNIQUE_KEY') }} from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + 'currency', + 'nzd', + ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY') }} + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('dedup_exchange_rate_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data.{{ quote('_AIRBYTE_UNIQUE_KEY') }} = new_data_ids.{{ quote('_AIRBYTE_UNIQUE_KEY') }} + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('dedup_exchange_rate_ab3') }} as inc_data on 1 = 0 + where {{ quote('_AIRBYTE_ACTIVE_ROW') }} = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('dedup_exchange_rate_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('dedup_exchange_rate_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('dedup_exchange_rate_ab3') }} + -- dedup_exchange_rate from {{ source('test_normalization', 'airbyte_raw_dedup_exchange_rate') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + 'currency', + 'nzd', + ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY') }}, + id, + currency, + {{ quote('DATE') }}, + timestamp_col, + hkd_special___characters, + hkd_special___characters_1, + nzd, + usd, + {{ quote('DATE') }} as {{ quote('_AIRBYTE_START_AT') }}, + lag({{ quote('DATE') }}) over ( + partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) + order by + {{ quote('DATE') }} asc nulls last, + {{ quote('DATE') }} desc, + {{ quote('_AIRBYTE_EMITTED_AT') }} desc + ) as {{ quote('_AIRBYTE_END_AT') }}, + case when lag({{ quote('DATE') }}) over ( + partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) + order by + {{ quote('DATE') }} asc nulls last, + {{ quote('DATE') }} desc, + {{ quote('_AIRBYTE_EMITTED_AT') }} desc + ) is null then 1 else 0 end as {{ quote('_AIRBYTE_ACTIVE_ROW') }}, + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ quote('_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID') }} + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by {{ quote('_AIRBYTE_UNIQUE_KEY') }}, {{ quote('_AIRBYTE_START_AT') }}, {{ quote('_AIRBYTE_EMITTED_AT') }} + order by {{ quote('_AIRBYTE_AB_ID') }} + ) as {{ quote('_AIRBYTE_ROW_NUM') }}, + {{ dbt_utils.surrogate_key([ + quote('_AIRBYTE_UNIQUE_KEY'), + quote('_AIRBYTE_START_AT'), + quote('_AIRBYTE_EMITTED_AT') + ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}, + scd_data.* + from scd_data +) +select + {{ quote('_AIRBYTE_UNIQUE_KEY') }}, + {{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}, + id, + currency, + {{ quote('DATE') }}, + timestamp_col, + hkd_special___characters, + hkd_special___characters_1, + nzd, + usd, + {{ quote('_AIRBYTE_START_AT') }}, + {{ quote('_AIRBYTE_END_AT') }}, + {{ quote('_AIRBYTE_ACTIVE_ROW') }}, + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }}, + {{ quote('_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID') }} +from dedup_data where {{ quote('_AIRBYTE_ROW_NUM') }} = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..9d79d2cbac72 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,103 @@ +{{ config( + unique_key = "{{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('renamed_dedup_cdc_excluded_ab3') }} + -- renamed_dedup_cdc_excluded from {{ source('test_normalization', 'airbyte_raw_renamed_dedup_cdc_excluded') }} + where 1 = 1 + {{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} +), +new_data_ids as ( + -- build a subset of {{ quote('_AIRBYTE_UNIQUE_KEY') }} from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY') }} + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('renamed_dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data.{{ quote('_AIRBYTE_UNIQUE_KEY') }} = new_data_ids.{{ quote('_AIRBYTE_UNIQUE_KEY') }} + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('renamed_dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 + where {{ quote('_AIRBYTE_ACTIVE_ROW') }} = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('renamed_dedup_cdc_excluded_ab3') }} + -- renamed_dedup_cdc_excluded from {{ source('test_normalization', 'airbyte_raw_renamed_dedup_cdc_excluded') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY') }}, + id, + {{ quote('_AIRBYTE_EMITTED_AT') }} as {{ quote('_AIRBYTE_START_AT') }}, + lag({{ quote('_AIRBYTE_EMITTED_AT') }}) over ( + partition by id + order by + {{ quote('_AIRBYTE_EMITTED_AT') }} asc nulls last, + {{ quote('_AIRBYTE_EMITTED_AT') }} desc, + {{ quote('_AIRBYTE_EMITTED_AT') }} desc + ) as {{ quote('_AIRBYTE_END_AT') }}, + case when lag({{ quote('_AIRBYTE_EMITTED_AT') }}) over ( + partition by id + order by + {{ quote('_AIRBYTE_EMITTED_AT') }} asc nulls last, + {{ quote('_AIRBYTE_EMITTED_AT') }} desc, + {{ quote('_AIRBYTE_EMITTED_AT') }} desc + ) is null then 1 else 0 end as {{ quote('_AIRBYTE_ACTIVE_ROW') }}, + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ quote('_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID') }} + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by {{ quote('_AIRBYTE_UNIQUE_KEY') }}, {{ quote('_AIRBYTE_START_AT') }}, {{ quote('_AIRBYTE_EMITTED_AT') }} + order by {{ quote('_AIRBYTE_AB_ID') }} + ) as {{ quote('_AIRBYTE_ROW_NUM') }}, + {{ dbt_utils.surrogate_key([ + quote('_AIRBYTE_UNIQUE_KEY'), + quote('_AIRBYTE_START_AT'), + quote('_AIRBYTE_EMITTED_AT') + ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}, + scd_data.* + from scd_data +) +select + {{ quote('_AIRBYTE_UNIQUE_KEY') }}, + {{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}, + id, + {{ quote('_AIRBYTE_START_AT') }}, + {{ quote('_AIRBYTE_END_AT') }}, + {{ quote('_AIRBYTE_ACTIVE_ROW') }}, + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }}, + {{ quote('_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID') }} +from dedup_data where {{ quote('_AIRBYTE_ROW_NUM') }} = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql new file mode 100644 index 000000000000..20be5269fbc9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -0,0 +1,23 @@ +{{ config( + unique_key = "{{ quote('_AIRBYTE_UNIQUE_KEY') }}", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + {{ quote('_AIRBYTE_UNIQUE_KEY') }}, + id, + name, + {{ quote('_AB_CDC_LSN') }}, + {{ quote('_AB_CDC_UPDATED_AT') }}, + {{ quote('_AB_CDC_DELETED_AT') }}, + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }}, + {{ quote('_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID') }} +from {{ ref('dedup_cdc_excluded_scd') }} +-- dedup_cdc_excluded from {{ source('test_normalization', 'airbyte_raw_dedup_cdc_excluded') }} +where 1 = 1 +and {{ quote('_AIRBYTE_ACTIVE_ROW') }} = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..d130087cf48a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,26 @@ +{{ config( + unique_key = "{{ quote('_AIRBYTE_UNIQUE_KEY') }}", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + {{ quote('_AIRBYTE_UNIQUE_KEY') }}, + id, + currency, + {{ quote('DATE') }}, + timestamp_col, + hkd_special___characters, + hkd_special___characters_1, + nzd, + usd, + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }}, + {{ quote('_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID') }} +from {{ ref('dedup_exchange_rate_scd') }} +-- dedup_exchange_rate from {{ source('test_normalization', 'airbyte_raw_dedup_exchange_rate') }} +where 1 = 1 +and {{ quote('_AIRBYTE_ACTIVE_ROW') }} = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..c87e205f2a32 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql @@ -0,0 +1,24 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + id, + currency, + {{ quote('DATE') }}, + timestamp_col, + hkd_special___characters, + hkd_special___characters_1, + nzd, + usd, + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }}, + {{ quote('_AIRBYTE_EXCHANGE_RATE_HASHID') }} +from {{ ref('exchange_rate_ab3') }} +-- exchange_rate from {{ source('test_normalization', 'airbyte_raw_exchange_rate') }} +where 1 = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..40bf1d0d0013 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,19 @@ +{{ config( + unique_key = "{{ quote('_AIRBYTE_UNIQUE_KEY') }}", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + {{ quote('_AIRBYTE_UNIQUE_KEY') }}, + id, + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }}, + {{ quote('_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID') }} +from {{ ref('renamed_dedup_cdc_excluded_scd') }} +-- renamed_dedup_cdc_excluded from {{ source('test_normalization', 'airbyte_raw_renamed_dedup_cdc_excluded') }} +where 1 = 1 +and {{ quote('_AIRBYTE_ACTIVE_ROW') }} = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..506d15538add --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,113 @@ +{{ config( + unique_key = "{{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('pos_dedup_cdcx_ab3') }} + -- pos_dedup_cdcx from {{ source('test_normalization', 'airbyte_raw_pos_dedup_cdcx') }} + where 1 = 1 + {{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} +), +new_data_ids as ( + -- build a subset of {{ quote('_AIRBYTE_UNIQUE_KEY') }} from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY') }} + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('pos_dedup_cdcx_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data.{{ quote('_AIRBYTE_UNIQUE_KEY') }} = new_data_ids.{{ quote('_AIRBYTE_UNIQUE_KEY') }} + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('pos_dedup_cdcx_ab3') }} as inc_data on 1 = 0 + where {{ quote('_AIRBYTE_ACTIVE_ROW') }} = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('pos_dedup_cdcx_ab3') }} + -- pos_dedup_cdcx from {{ source('test_normalization', 'airbyte_raw_pos_dedup_cdcx') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY') }}, + id, + name, + {{ quote('_AB_CDC_LSN') }}, + {{ quote('_AB_CDC_UPDATED_AT') }}, + {{ quote('_AB_CDC_DELETED_AT') }}, + {{ quote('_AB_CDC_LOG_POS') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }} as {{ quote('_AIRBYTE_START_AT') }}, + lag({{ quote('_AIRBYTE_EMITTED_AT') }}) over ( + partition by id + order by + {{ quote('_AIRBYTE_EMITTED_AT') }} asc nulls last, + {{ quote('_AIRBYTE_EMITTED_AT') }} desc, + {{ quote('_AIRBYTE_EMITTED_AT') }} desc, {{ quote('_AB_CDC_UPDATED_AT') }} desc, {{ quote('_AB_CDC_LOG_POS') }} desc + ) as {{ quote('_AIRBYTE_END_AT') }}, + case when lag({{ quote('_AIRBYTE_EMITTED_AT') }}) over ( + partition by id + order by + {{ quote('_AIRBYTE_EMITTED_AT') }} asc nulls last, + {{ quote('_AIRBYTE_EMITTED_AT') }} desc, + {{ quote('_AIRBYTE_EMITTED_AT') }} desc, {{ quote('_AB_CDC_UPDATED_AT') }} desc, {{ quote('_AB_CDC_LOG_POS') }} desc + ) is null and {{ quote('_AB_CDC_DELETED_AT') }} is null then 1 else 0 end as {{ quote('_AIRBYTE_ACTIVE_ROW') }}, + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ quote('_AIRBYTE_POS_DEDUP_CDCX_HASHID') }} + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by {{ quote('_AIRBYTE_UNIQUE_KEY') }}, {{ quote('_AIRBYTE_START_AT') }}, {{ quote('_AIRBYTE_EMITTED_AT') }}, cast({{ quote('_AB_CDC_DELETED_AT') }} as {{ dbt_utils.type_string() }}), cast({{ quote('_AB_CDC_UPDATED_AT') }} as {{ dbt_utils.type_string() }}), cast({{ quote('_AB_CDC_LOG_POS') }} as {{ dbt_utils.type_string() }}) + order by {{ quote('_AIRBYTE_AB_ID') }} + ) as {{ quote('_AIRBYTE_ROW_NUM') }}, + {{ dbt_utils.surrogate_key([ + quote('_AIRBYTE_UNIQUE_KEY'), + quote('_AIRBYTE_START_AT'), + quote('_AIRBYTE_EMITTED_AT'), quote('_AB_CDC_DELETED_AT'), quote('_AB_CDC_UPDATED_AT'), quote('_AB_CDC_LOG_POS') + ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}, + scd_data.* + from scd_data +) +select + {{ quote('_AIRBYTE_UNIQUE_KEY') }}, + {{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}, + id, + name, + {{ quote('_AB_CDC_LSN') }}, + {{ quote('_AB_CDC_UPDATED_AT') }}, + {{ quote('_AB_CDC_DELETED_AT') }}, + {{ quote('_AB_CDC_LOG_POS') }}, + {{ quote('_AIRBYTE_START_AT') }}, + {{ quote('_AIRBYTE_END_AT') }}, + {{ quote('_AIRBYTE_ACTIVE_ROW') }}, + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }}, + {{ quote('_AIRBYTE_POS_DEDUP_CDCX_HASHID') }} +from dedup_data where {{ quote('_AIRBYTE_ROW_NUM') }} = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index 9578d423ecf4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - name, - column___with__quotes, - {{ quote('_AB_CDC_LSN') }}, - {{ quote('_AB_CDC_UPDATED_AT') }}, - {{ quote('_AB_CDC_DELETED_AT') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }}, - {{ quote('_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID') }} -from {{ ref('dedup_cdc_excluded_scd') }} --- dedup_cdc_excluded from {{ source('test_normalization', 'airbyte_raw_dedup_cdc_excluded') }} -where "_AIRBYTE_ACTIVE_ROW" = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index 167dd8604a46..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - column___with__quotes, - {{ quote('_AB_CDC_LSN') }}, - {{ quote('_AB_CDC_UPDATED_AT') }}, - {{ quote('_AB_CDC_DELETED_AT') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }} as {{ quote('_AIRBYTE_START_AT') }}, - lag({{ quote('_AIRBYTE_EMITTED_AT') }}) over ( - partition by id - order by {{ quote('_AIRBYTE_EMITTED_AT') }} asc nulls last, {{ quote('_AIRBYTE_EMITTED_AT') }} desc, {{ quote('_AIRBYTE_EMITTED_AT') }} desc - ) as {{ quote('_AIRBYTE_END_AT') }}, - case when lag({{ quote('_AIRBYTE_EMITTED_AT') }}) over ( - partition by id - order by {{ quote('_AIRBYTE_EMITTED_AT') }} asc nulls last, {{ quote('_AIRBYTE_EMITTED_AT') }} desc, {{ quote('_AIRBYTE_EMITTED_AT') }} desc, {{ quote('_AB_CDC_UPDATED_AT') }} desc - ) is null and {{ quote('_AB_CDC_DELETED_AT') }} is null then 1 else 0 end as {{ quote('_AIRBYTE_ACTIVE_ROW') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }}, - {{ quote('_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID') }} -from {{ ref('dedup_cdc_excluded_ab4') }} --- dedup_cdc_excluded from {{ source('test_normalization', 'airbyte_raw_dedup_cdc_excluded') }} -where "_AIRBYTE_ROW_NUM" = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql deleted file mode 100644 index d423d5bf7f1a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - currency, - {{ quote('DATE') }}, - timestamp_col, - hkd_special___characters, - hkd_special___characters_1, - nzd, - usd, - {{ quote('_AIRBYTE_EMITTED_AT') }}, - {{ quote('_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID') }} -from {{ ref('dedup_exchange_rate_scd') }} --- dedup_exchange_rate from {{ source('test_normalization', 'airbyte_raw_dedup_exchange_rate') }} -where "_AIRBYTE_ACTIVE_ROW" = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql deleted file mode 100644 index 00050f79d2e6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql +++ /dev/null @@ -1,26 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - currency, - {{ quote('DATE') }}, - timestamp_col, - hkd_special___characters, - hkd_special___characters_1, - nzd, - usd, - {{ quote('DATE') }} as {{ quote('_AIRBYTE_START_AT') }}, - lag({{ quote('DATE') }}) over ( - partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) - order by {{ quote('DATE') }} asc nulls last, {{ quote('DATE') }} desc, {{ quote('_AIRBYTE_EMITTED_AT') }} desc - ) as {{ quote('_AIRBYTE_END_AT') }}, - case when lag({{ quote('DATE') }}) over ( - partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) - order by {{ quote('DATE') }} asc nulls last, {{ quote('DATE') }} desc, {{ quote('_AIRBYTE_EMITTED_AT') }} desc - ) is null then 1 else 0 end as {{ quote('_AIRBYTE_ACTIVE_ROW') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }}, - {{ quote('_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID') }} -from {{ ref('dedup_exchange_rate_ab4') }} --- dedup_exchange_rate from {{ source('test_normalization', 'airbyte_raw_dedup_exchange_rate') }} -where "_AIRBYTE_ROW_NUM" = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql deleted file mode 100644 index 5ca8bf28bd17..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - currency, - {{ quote('DATE') }}, - timestamp_col, - hkd_special___characters, - hkd_special___characters_1, - nzd, - usd, - {{ quote('_AIRBYTE_EMITTED_AT') }}, - {{ quote('_AIRBYTE_EXCHANGE_RATE_HASHID') }} -from {{ ref('exchange_rate_ab3') }} --- exchange_rate from {{ source('test_normalization', 'airbyte_raw_exchange_rate') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql index e1e38ef48412..26ed51d88c7b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -1,15 +1,23 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = "{{ quote('_AIRBYTE_UNIQUE_KEY') }}", + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select + {{ quote('_AIRBYTE_UNIQUE_KEY') }}, id, name, {{ quote('_AB_CDC_LSN') }}, {{ quote('_AB_CDC_UPDATED_AT') }}, {{ quote('_AB_CDC_DELETED_AT') }}, {{ quote('_AB_CDC_LOG_POS') }}, + {{ quote('_AIRBYTE_AB_ID') }}, {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }}, {{ quote('_AIRBYTE_POS_DEDUP_CDCX_HASHID') }} from {{ ref('pos_dedup_cdcx_scd') }} -- pos_dedup_cdcx from {{ source('test_normalization', 'airbyte_raw_pos_dedup_cdcx') }} -where "_AIRBYTE_ACTIVE_ROW" = 1 +where 1 = 1 +and {{ quote('_AIRBYTE_ACTIVE_ROW') }} = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index 4abe4a621ad5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - {{ quote('_AB_CDC_LSN') }}, - {{ quote('_AB_CDC_UPDATED_AT') }}, - {{ quote('_AB_CDC_DELETED_AT') }}, - {{ quote('_AB_CDC_LOG_POS') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }} as {{ quote('_AIRBYTE_START_AT') }}, - lag({{ quote('_AIRBYTE_EMITTED_AT') }}) over ( - partition by id - order by {{ quote('_AIRBYTE_EMITTED_AT') }} asc nulls last, {{ quote('_AIRBYTE_EMITTED_AT') }} desc, {{ quote('_AIRBYTE_EMITTED_AT') }} desc - ) as {{ quote('_AIRBYTE_END_AT') }}, - case when lag({{ quote('_AIRBYTE_EMITTED_AT') }}) over ( - partition by id - order by {{ quote('_AIRBYTE_EMITTED_AT') }} asc nulls last, {{ quote('_AIRBYTE_EMITTED_AT') }} desc, {{ quote('_AIRBYTE_EMITTED_AT') }} desc, {{ quote('_AB_CDC_UPDATED_AT') }} desc, {{ quote('_AB_CDC_LOG_POS') }} desc - ) is null and {{ quote('_AB_CDC_DELETED_AT') }} is null then 1 else 0 end as {{ quote('_AIRBYTE_ACTIVE_ROW') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }}, - {{ quote('_AIRBYTE_POS_DEDUP_CDCX_HASHID') }} -from {{ ref('pos_dedup_cdcx_ab4') }} --- pos_dedup_cdcx from {{ source('test_normalization', 'airbyte_raw_pos_dedup_cdcx') }} -where "_AIRBYTE_ROW_NUM" = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..f9575722cc41 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,20 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + 'name', + quote('_AB_CDC_LSN'), + quote('_AB_CDC_UPDATED_AT'), + quote('_AB_CDC_DELETED_AT'), + ]) }} as {{ quote('_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID') }}, + tmp.* +from {{ ref('dedup_cdc_excluded_ab2') }} tmp +-- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..48905c777a2d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,23 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + 'currency', + quote('DATE'), + 'timestamp_col', + 'hkd_special___characters', + 'hkd_special___characters_1', + 'nzd', + 'usd', + ]) }} as {{ quote('_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID') }}, + tmp.* +from {{ ref('dedup_exchange_rate_ab2') }} tmp +-- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql new file mode 100644 index 000000000000..52cf027c3895 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -0,0 +1,20 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + 'name', + quote('_AB_CDC_LSN'), + quote('_AB_CDC_UPDATED_AT'), + quote('_AB_CDC_DELETED_AT'), + quote('_AB_CDC_LOG_POS'), + ]) }} as {{ quote('_AIRBYTE_POS_DEDUP_CDCX_HASHID') }}, + tmp.* +from {{ ref('pos_dedup_cdcx_ab2') }} tmp +-- pos_dedup_cdcx +where 1 = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..7555ff5d1efd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,16 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as {{ quote('_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID') }}, + tmp.* +from {{ ref('renamed_dedup_cdc_excluded_ab2') }} tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/sources.yml index 954680f83f0e..c6b4b6023ba5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/sources.yml @@ -10,3 +10,4 @@ sources: - name: airbyte_raw_dedup_exchange_rate - name: airbyte_raw_exchange_rate - name: airbyte_raw_pos_dedup_cdcx + - name: airbyte_raw_renamed_dedup_cdc_excluded diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..bd51b214ebc6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,93 @@ + + + create table test_normalization.dedup_cdc_excluded_scd__dbt_tmp + + as + +with + +input_data as ( + select * + from test_normalization.dedup_cdc_excluded_ab3 + -- dedup_cdc_excluded from test_normalization.airbyte_raw_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + ora_hash( + + id + + ) as "_AIRBYTE_UNIQUE_KEY", + id, + name, + "_AB_CDC_LSN", + "_AB_CDC_UPDATED_AT", + "_AB_CDC_DELETED_AT", + "_AIRBYTE_EMITTED_AT" as "_AIRBYTE_START_AT", + lag("_AIRBYTE_EMITTED_AT") over ( + partition by id + order by + "_AIRBYTE_EMITTED_AT" asc nulls last, + "_AIRBYTE_EMITTED_AT" desc, + "_AIRBYTE_EMITTED_AT" desc, "_AB_CDC_UPDATED_AT" desc + ) as "_AIRBYTE_END_AT", + case when lag("_AIRBYTE_EMITTED_AT") over ( + partition by id + order by + "_AIRBYTE_EMITTED_AT" asc nulls last, + "_AIRBYTE_EMITTED_AT" desc, + "_AIRBYTE_EMITTED_AT" desc, "_AB_CDC_UPDATED_AT" desc + ) is null and "_AB_CDC_DELETED_AT" is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_START_AT", "_AIRBYTE_EMITTED_AT", cast("_AB_CDC_DELETED_AT" as varchar2(4000)), cast("_AB_CDC_UPDATED_AT" as varchar2(4000)) + order by "_AIRBYTE_AB_ID" + ) as "_AIRBYTE_ROW_NUM", + ora_hash( + + "_AIRBYTE_UNIQUE_KEY" || '~' || + + + "_AIRBYTE_START_AT" || '~' || + + + "_AIRBYTE_EMITTED_AT" || '~' || + + + "_AB_CDC_DELETED_AT" || '~' || + + + "_AB_CDC_UPDATED_AT" + + ) as "_AIRBYTE_UNIQUE_KEY_SCD", + scd_data.* + from scd_data +) +select + "_AIRBYTE_UNIQUE_KEY", + "_AIRBYTE_UNIQUE_KEY_SCD", + id, + name, + "_AB_CDC_LSN", + "_AB_CDC_UPDATED_AT", + "_AB_CDC_DELETED_AT", + "_AIRBYTE_START_AT", + "_AIRBYTE_END_AT", + "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" +from dedup_data where "_AIRBYTE_ROW_NUM" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..e6c91980fea5 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,99 @@ + + + create table test_normalization.dedup_exchange_rate_scd__dbt_tmp + + as + +with + +input_data as ( + select * + from test_normalization.dedup_exchange_rate_ab3 + -- dedup_exchange_rate from test_normalization.airbyte_raw_dedup_exchange_rate +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + ora_hash( + + id || '~' || + + + currency || '~' || + + + nzd + + ) as "_AIRBYTE_UNIQUE_KEY", + id, + currency, + "DATE", + timestamp_col, + hkd_special___characters, + hkd_special___characters_1, + nzd, + usd, + "DATE" as "_AIRBYTE_START_AT", + lag("DATE") over ( + partition by id, currency, cast(nzd as varchar2(4000)) + order by + "DATE" asc nulls last, + "DATE" desc, + "_AIRBYTE_EMITTED_AT" desc + ) as "_AIRBYTE_END_AT", + case when lag("DATE") over ( + partition by id, currency, cast(nzd as varchar2(4000)) + order by + "DATE" asc nulls last, + "DATE" desc, + "_AIRBYTE_EMITTED_AT" desc + ) is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID" + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_START_AT", "_AIRBYTE_EMITTED_AT" + order by "_AIRBYTE_AB_ID" + ) as "_AIRBYTE_ROW_NUM", + ora_hash( + + "_AIRBYTE_UNIQUE_KEY" || '~' || + + + "_AIRBYTE_START_AT" || '~' || + + + "_AIRBYTE_EMITTED_AT" + + ) as "_AIRBYTE_UNIQUE_KEY_SCD", + scd_data.* + from scd_data +) +select + "_AIRBYTE_UNIQUE_KEY", + "_AIRBYTE_UNIQUE_KEY_SCD", + id, + currency, + "DATE", + timestamp_col, + hkd_special___characters, + hkd_special___characters_1, + nzd, + usd, + "_AIRBYTE_START_AT", + "_AIRBYTE_END_AT", + "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID" +from dedup_data where "_AIRBYTE_ROW_NUM" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..c8d9914d17d2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,79 @@ + + + create table test_normalization.renamed_dedup_cdc_excluded_scd__dbt_tmp + + as + +with + +input_data as ( + select * + from test_normalization.renamed_dedup_cdc_excluded_ab3 + -- renamed_dedup_cdc_excluded from test_normalization.airbyte_raw_renamed_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + ora_hash( + + id + + ) as "_AIRBYTE_UNIQUE_KEY", + id, + "_AIRBYTE_EMITTED_AT" as "_AIRBYTE_START_AT", + lag("_AIRBYTE_EMITTED_AT") over ( + partition by id + order by + "_AIRBYTE_EMITTED_AT" asc nulls last, + "_AIRBYTE_EMITTED_AT" desc, + "_AIRBYTE_EMITTED_AT" desc + ) as "_AIRBYTE_END_AT", + case when lag("_AIRBYTE_EMITTED_AT") over ( + partition by id + order by + "_AIRBYTE_EMITTED_AT" asc nulls last, + "_AIRBYTE_EMITTED_AT" desc, + "_AIRBYTE_EMITTED_AT" desc + ) is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_START_AT", "_AIRBYTE_EMITTED_AT" + order by "_AIRBYTE_AB_ID" + ) as "_AIRBYTE_ROW_NUM", + ora_hash( + + "_AIRBYTE_UNIQUE_KEY" || '~' || + + + "_AIRBYTE_START_AT" || '~' || + + + "_AIRBYTE_EMITTED_AT" + + ) as "_AIRBYTE_UNIQUE_KEY_SCD", + scd_data.* + from scd_data +) +select + "_AIRBYTE_UNIQUE_KEY", + "_AIRBYTE_UNIQUE_KEY_SCD", + id, + "_AIRBYTE_START_AT", + "_AIRBYTE_END_AT", + "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" +from dedup_data where "_AIRBYTE_ROW_NUM" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql new file mode 100644 index 000000000000..bf4c8a11c07b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -0,0 +1,24 @@ + + + create table test_normalization.dedup_cdc_excluded__dbt_tmp + + as + +-- Final base SQL model +select + "_AIRBYTE_UNIQUE_KEY", + id, + name, + "_AB_CDC_LSN", + "_AB_CDC_UPDATED_AT", + "_AB_CDC_DELETED_AT", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" +from test_normalization.dedup_cdc_excluded_scd +-- dedup_cdc_excluded from test_normalization.airbyte_raw_dedup_cdc_excluded +where 1 = 1 +and "_AIRBYTE_ACTIVE_ROW" = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..f67f7384cf17 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,27 @@ + + + create table test_normalization.dedup_exchange_rate__dbt_tmp + + as + +-- Final base SQL model +select + "_AIRBYTE_UNIQUE_KEY", + id, + currency, + "DATE", + timestamp_col, + hkd_special___characters, + hkd_special___characters_1, + nzd, + usd, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID" +from test_normalization.dedup_exchange_rate_scd +-- dedup_exchange_rate from test_normalization.airbyte_raw_dedup_exchange_rate +where 1 = 1 +and "_AIRBYTE_ACTIVE_ROW" = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..9d77656b65a9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -0,0 +1,114 @@ + + + create table test_normalization.exchange_rate__dbt_tmp + + as + +with dbt__cte__exchange_rate_ab1__ as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value("_AIRBYTE_DATA", '$."id"') as id, + json_value("_AIRBYTE_DATA", '$."currency"') as currency, + json_value("_AIRBYTE_DATA", '$."date"') as "DATE", + json_value("_AIRBYTE_DATA", '$."timestamp_col"') as timestamp_col, + json_value("_AIRBYTE_DATA", '$."HKD@spéçiäl & characters"') as hkd_special___characters, + json_value("_AIRBYTE_DATA", '$."HKD_special___characters"') as hkd_special___characters_1, + json_value("_AIRBYTE_DATA", '$."NZD"') as nzd, + json_value("_AIRBYTE_DATA", '$."USD"') as usd, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from test_normalization.airbyte_raw_exchange_rate +-- exchange_rate +where 1 = 1 + +), dbt__cte__exchange_rate_ab2__ as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + numeric +) as id, + cast(currency as varchar2(4000)) as currency, + cast(nullif("DATE", '') as + varchar2(4000) +) as "DATE", + cast(nullif(timestamp_col, '') as + varchar2(4000) +) as timestamp_col, + cast(hkd_special___characters as + float +) as hkd_special___characters, + cast(hkd_special___characters_1 as varchar2(4000)) as hkd_special___characters_1, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from dbt__cte__exchange_rate_ab1__ +-- exchange_rate +where 1 = 1 + +), dbt__cte__exchange_rate_ab3__ as ( + +-- SQL model to build a hash column based on the values of this record +select + ora_hash( + + id || '~' || + + + currency || '~' || + + + "DATE" || '~' || + + + timestamp_col || '~' || + + + hkd_special___characters || '~' || + + + hkd_special___characters_1 || '~' || + + + nzd || '~' || + + + usd + + ) as "_AIRBYTE_EXCHANGE_RATE_HASHID", + tmp.* +from dbt__cte__exchange_rate_ab2__ tmp +-- exchange_rate +where 1 = 1 + +)-- Final base SQL model +select + id, + currency, + "DATE", + timestamp_col, + hkd_special___characters, + hkd_special___characters_1, + nzd, + usd, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_EXCHANGE_RATE_HASHID" +from dbt__cte__exchange_rate_ab3__ +-- exchange_rate from test_normalization.airbyte_raw_exchange_rate +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..feac3368a6ca --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,20 @@ + + + create table test_normalization.renamed_dedup_cdc_excluded__dbt_tmp + + as + +-- Final base SQL model +select + "_AIRBYTE_UNIQUE_KEY", + id, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" +from test_normalization.renamed_dedup_cdc_excluded_scd +-- renamed_dedup_cdc_excluded from test_normalization.airbyte_raw_renamed_dedup_cdc_excluded +where 1 = 1 +and "_AIRBYTE_ACTIVE_ROW" = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..c184fd31e2d3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,98 @@ + + + create table test_normalization.pos_dedup_cdcx_scd__dbt_tmp + + as + +with + +input_data as ( + select * + from test_normalization.pos_dedup_cdcx_ab3 + -- pos_dedup_cdcx from test_normalization.airbyte_raw_pos_dedup_cdcx +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + ora_hash( + + id + + ) as "_AIRBYTE_UNIQUE_KEY", + id, + name, + "_AB_CDC_LSN", + "_AB_CDC_UPDATED_AT", + "_AB_CDC_DELETED_AT", + "_AB_CDC_LOG_POS", + "_AIRBYTE_EMITTED_AT" as "_AIRBYTE_START_AT", + lag("_AIRBYTE_EMITTED_AT") over ( + partition by id + order by + "_AIRBYTE_EMITTED_AT" asc nulls last, + "_AIRBYTE_EMITTED_AT" desc, + "_AIRBYTE_EMITTED_AT" desc, "_AB_CDC_UPDATED_AT" desc, "_AB_CDC_LOG_POS" desc + ) as "_AIRBYTE_END_AT", + case when lag("_AIRBYTE_EMITTED_AT") over ( + partition by id + order by + "_AIRBYTE_EMITTED_AT" asc nulls last, + "_AIRBYTE_EMITTED_AT" desc, + "_AIRBYTE_EMITTED_AT" desc, "_AB_CDC_UPDATED_AT" desc, "_AB_CDC_LOG_POS" desc + ) is null and "_AB_CDC_DELETED_AT" is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + "_AIRBYTE_POS_DEDUP_CDCX_HASHID" + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_START_AT", "_AIRBYTE_EMITTED_AT", cast("_AB_CDC_DELETED_AT" as varchar2(4000)), cast("_AB_CDC_UPDATED_AT" as varchar2(4000)), cast("_AB_CDC_LOG_POS" as varchar2(4000)) + order by "_AIRBYTE_AB_ID" + ) as "_AIRBYTE_ROW_NUM", + ora_hash( + + "_AIRBYTE_UNIQUE_KEY" || '~' || + + + "_AIRBYTE_START_AT" || '~' || + + + "_AIRBYTE_EMITTED_AT" || '~' || + + + "_AB_CDC_DELETED_AT" || '~' || + + + "_AB_CDC_UPDATED_AT" || '~' || + + + "_AB_CDC_LOG_POS" + + ) as "_AIRBYTE_UNIQUE_KEY_SCD", + scd_data.* + from scd_data +) +select + "_AIRBYTE_UNIQUE_KEY", + "_AIRBYTE_UNIQUE_KEY_SCD", + id, + name, + "_AB_CDC_LSN", + "_AB_CDC_UPDATED_AT", + "_AB_CDC_DELETED_AT", + "_AB_CDC_LOG_POS", + "_AIRBYTE_START_AT", + "_AIRBYTE_END_AT", + "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_POS_DEDUP_CDCX_HASHID" +from dedup_data where "_AIRBYTE_ROW_NUM" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql new file mode 100644 index 000000000000..d3fd87ee5576 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -0,0 +1,25 @@ + + + create table test_normalization.pos_dedup_cdcx__dbt_tmp + + as + +-- Final base SQL model +select + "_AIRBYTE_UNIQUE_KEY", + id, + name, + "_AB_CDC_LSN", + "_AB_CDC_UPDATED_AT", + "_AB_CDC_DELETED_AT", + "_AB_CDC_LOG_POS", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_POS_DEDUP_CDCX_HASHID" +from test_normalization.pos_dedup_cdcx_scd +-- pos_dedup_cdcx from test_normalization.airbyte_raw_pos_dedup_cdcx +where 1 = 1 +and "_AIRBYTE_ACTIVE_ROW" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..30135e3c86ec --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,72 @@ + + create view test_normalization.dedup_cdc_excluded_ab3__dbt_tmp as + +with dbt__cte__dedup_cdc_excluded_ab1__ as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value("_AIRBYTE_DATA", '$."id"') as id, + json_value("_AIRBYTE_DATA", '$."name"') as name, + json_value("_AIRBYTE_DATA", '$."_ab_cdc_lsn"') as "_AB_CDC_LSN", + json_value("_AIRBYTE_DATA", '$."_ab_cdc_updated_at"') as "_AB_CDC_UPDATED_AT", + json_value("_AIRBYTE_DATA", '$."_ab_cdc_deleted_at"') as "_AB_CDC_DELETED_AT", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from test_normalization.airbyte_raw_dedup_cdc_excluded +-- dedup_cdc_excluded +where 1 = 1 + +), dbt__cte__dedup_cdc_excluded_ab2__ as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + numeric +) as id, + cast(name as varchar2(4000)) as name, + cast("_AB_CDC_LSN" as + float +) as "_AB_CDC_LSN", + cast("_AB_CDC_UPDATED_AT" as + float +) as "_AB_CDC_UPDATED_AT", + cast("_AB_CDC_DELETED_AT" as + float +) as "_AB_CDC_DELETED_AT", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from dbt__cte__dedup_cdc_excluded_ab1__ +-- dedup_cdc_excluded +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + ora_hash( + + id || '~' || + + + name || '~' || + + + "_AB_CDC_LSN" || '~' || + + + "_AB_CDC_UPDATED_AT" || '~' || + + + "_AB_CDC_DELETED_AT" + + ) as "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID", + tmp.* +from dbt__cte__dedup_cdc_excluded_ab2__ tmp +-- dedup_cdc_excluded +where 1 = 1 + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..527196251304 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,91 @@ + + create view test_normalization.dedup_exchange_rate_ab3__dbt_tmp as + +with dbt__cte__dedup_exchange_rate_ab1__ as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value("_AIRBYTE_DATA", '$."id"') as id, + json_value("_AIRBYTE_DATA", '$."currency"') as currency, + json_value("_AIRBYTE_DATA", '$."date"') as "DATE", + json_value("_AIRBYTE_DATA", '$."timestamp_col"') as timestamp_col, + json_value("_AIRBYTE_DATA", '$."HKD@spéçiäl & characters"') as hkd_special___characters, + json_value("_AIRBYTE_DATA", '$."HKD_special___characters"') as hkd_special___characters_1, + json_value("_AIRBYTE_DATA", '$."NZD"') as nzd, + json_value("_AIRBYTE_DATA", '$."USD"') as usd, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from test_normalization.airbyte_raw_dedup_exchange_rate +-- dedup_exchange_rate +where 1 = 1 + +), dbt__cte__dedup_exchange_rate_ab2__ as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + numeric +) as id, + cast(currency as varchar2(4000)) as currency, + cast(nullif("DATE", '') as + varchar2(4000) +) as "DATE", + cast(nullif(timestamp_col, '') as + varchar2(4000) +) as timestamp_col, + cast(hkd_special___characters as + float +) as hkd_special___characters, + cast(hkd_special___characters_1 as varchar2(4000)) as hkd_special___characters_1, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from dbt__cte__dedup_exchange_rate_ab1__ +-- dedup_exchange_rate +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + ora_hash( + + id || '~' || + + + currency || '~' || + + + "DATE" || '~' || + + + timestamp_col || '~' || + + + hkd_special___characters || '~' || + + + hkd_special___characters_1 || '~' || + + + nzd || '~' || + + + usd + + ) as "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID", + tmp.* +from dbt__cte__dedup_exchange_rate_ab2__ tmp +-- dedup_exchange_rate +where 1 = 1 + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql new file mode 100644 index 000000000000..e1a0af83a632 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -0,0 +1,76 @@ + + create view test_normalization.pos_dedup_cdcx_ab3__dbt_tmp as + +with dbt__cte__pos_dedup_cdcx_ab1__ as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value("_AIRBYTE_DATA", '$."id"') as id, + json_value("_AIRBYTE_DATA", '$."name"') as name, + json_value("_AIRBYTE_DATA", '$."_ab_cdc_lsn"') as "_AB_CDC_LSN", + json_value("_AIRBYTE_DATA", '$."_ab_cdc_updated_at"') as "_AB_CDC_UPDATED_AT", + json_value("_AIRBYTE_DATA", '$."_ab_cdc_deleted_at"') as "_AB_CDC_DELETED_AT", + json_value("_AIRBYTE_DATA", '$."_ab_cdc_log_pos"') as "_AB_CDC_LOG_POS", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from test_normalization.airbyte_raw_pos_dedup_cdcx +-- pos_dedup_cdcx +where 1 = 1 +), dbt__cte__pos_dedup_cdcx_ab2__ as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + numeric +) as id, + cast(name as varchar2(4000)) as name, + cast("_AB_CDC_LSN" as + float +) as "_AB_CDC_LSN", + cast("_AB_CDC_UPDATED_AT" as + float +) as "_AB_CDC_UPDATED_AT", + cast("_AB_CDC_DELETED_AT" as + float +) as "_AB_CDC_DELETED_AT", + cast("_AB_CDC_LOG_POS" as + float +) as "_AB_CDC_LOG_POS", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from dbt__cte__pos_dedup_cdcx_ab1__ +-- pos_dedup_cdcx +where 1 = 1 +)-- SQL model to build a hash column based on the values of this record +select + ora_hash( + + id || '~' || + + + name || '~' || + + + "_AB_CDC_LSN" || '~' || + + + "_AB_CDC_UPDATED_AT" || '~' || + + + "_AB_CDC_DELETED_AT" || '~' || + + + "_AB_CDC_LOG_POS" + + ) as "_AIRBYTE_POS_DEDUP_CDCX_HASHID", + tmp.* +from dbt__cte__pos_dedup_cdcx_ab2__ tmp +-- pos_dedup_cdcx +where 1 = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..6ad67cd02a90 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,46 @@ + + create view test_normalization.renamed_dedup_cdc_excluded_ab3__dbt_tmp as + +with dbt__cte__renamed_dedup_cdc_excluded_ab1__ as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value("_AIRBYTE_DATA", '$."id"') as id, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from test_normalization.airbyte_raw_renamed_dedup_cdc_excluded +-- renamed_dedup_cdc_excluded +where 1 = 1 + +), dbt__cte__renamed_dedup_cdc_excluded_ab2__ as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + numeric +) as id, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from dbt__cte__renamed_dedup_cdc_excluded_ab1__ +-- renamed_dedup_cdc_excluded +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + ora_hash( + + id + + ) as "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID", + tmp.* +from dbt__cte__renamed_dedup_cdc_excluded_ab2__ tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql deleted file mode 100644 index 693121b8cdc5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_array_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path(_airbyte_data, 'conflict_stream_array') as conflict_stream_array, - _airbyte_emitted_at -from "postgres".test_normalization._airbyte_raw_conflict_stream_array as table_alias --- conflict_stream_array - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql deleted file mode 100644 index 7cb90ed2af78..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_array_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - varchar -) as "id", - conflict_stream_array, - _airbyte_emitted_at -from "postgres"._airbyte_test_normalization."conflict_stream_array_ab1" --- conflict_stream_array - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql deleted file mode 100644 index 5ba21b8a148f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_array_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast(conflict_stream_array as - varchar -), '') as - varchar -)) as _airbyte_conflict_stream_array_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."conflict_stream_array_ab2" tmp --- conflict_stream_array - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql deleted file mode 100644 index 6269ba0ab954..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql +++ /dev/null @@ -1,12 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_2_hashid, - jsonb_extract_path_text(conflict_stream_name, 'groups') as groups, - _airbyte_emitted_at -from "postgres".test_normalization."conflict_stream_name_conflict_stream_name" as table_alias -where conflict_stream_name is not null --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql deleted file mode 100644 index 671aa9e8df7d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_2_hashid, - cast(groups as - varchar -) as groups, - _airbyte_emitted_at -from "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab1" --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql deleted file mode 100644 index acdc2bce21b1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as - varchar -), '') || '-' || coalesce(cast(groups as - varchar -), '') as - varchar -)) as _airbyte_conflict_stream_name_3_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab2" tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql deleted file mode 100644 index 8d5e15e1cd45..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_name_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - - jsonb_extract_path(table_alias._airbyte_data, 'conflict_stream_name') - as conflict_stream_name, - _airbyte_emitted_at -from "postgres".test_normalization._airbyte_raw_conflict_stream_name as table_alias --- conflict_stream_name - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql deleted file mode 100644 index 5bbbb58c1832..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ /dev/null @@ -1,15 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_name_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - varchar -) as "id", - cast(conflict_stream_name as - jsonb -) as conflict_stream_name, - _airbyte_emitted_at -from "postgres"._airbyte_test_normalization."conflict_stream_name_ab1" --- conflict_stream_name - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql deleted file mode 100644 index 4d7cf4c3ae76..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_name_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast(conflict_stream_name as - varchar -), '') as - varchar -)) as _airbyte_conflict_stream_name_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."conflict_stream_name_ab2" tmp --- conflict_stream_name - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql deleted file mode 100644 index af9e0733f509..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_hashid, - - jsonb_extract_path(table_alias.conflict_stream_name, 'conflict_stream_name') - as conflict_stream_name, - _airbyte_emitted_at -from "postgres".test_normalization."conflict_stream_name" as table_alias -where conflict_stream_name is not null --- conflict_stream_name at conflict_stream_name/conflict_stream_name - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql deleted file mode 100644 index 3df5e1f853fa..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_hashid, - cast(conflict_stream_name as - jsonb -) as conflict_stream_name, - _airbyte_emitted_at -from "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab1" --- conflict_stream_name at conflict_stream_name/conflict_stream_name - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql deleted file mode 100644 index 255cc446e679..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_conflict_stream_name_hashid as - varchar -), '') || '-' || coalesce(cast(conflict_stream_name as - varchar -), '') as - varchar -)) as _airbyte_conflict_stream_name_2_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab2" tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql deleted file mode 100644 index c6c3552e6f57..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path_text(_airbyte_data, 'conflict_stream_scalar') as conflict_stream_scalar, - _airbyte_emitted_at -from "postgres".test_normalization._airbyte_raw_conflict_stream_scalar as table_alias --- conflict_stream_scalar - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql deleted file mode 100644 index 06644717b27d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ /dev/null @@ -1,15 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - varchar -) as "id", - cast(conflict_stream_scalar as - bigint -) as conflict_stream_scalar, - _airbyte_emitted_at -from "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab1" --- conflict_stream_scalar - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql deleted file mode 100644 index 7435231cc854..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast(conflict_stream_scalar as - varchar -), '') as - varchar -)) as _airbyte_conflict_stream_scalar_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab2" tmp --- conflict_stream_scalar - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql deleted file mode 100644 index a15d7c286441..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_nested_stre__nto_long_names_hashid, - jsonb_extract_path("partition", 'double_array_data') as double_array_data, - jsonb_extract_path("partition", 'DATA') as "DATA", - jsonb_extract_path("partition", 'column`_''with"_quotes') as "column`_'with""_quotes", - _airbyte_emitted_at -from "postgres".test_normalization."nested_stream_with_c__lting_into_long_names" as table_alias -where "partition" is not null --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql deleted file mode 100644 index 8beca5ca9421..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_nested_stre__nto_long_names_hashid, - double_array_data, - "DATA", - "column`_'with""_quotes", - _airbyte_emitted_at -from "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab1" --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql deleted file mode 100644 index 0d745ed71a40..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql +++ /dev/null @@ -1,20 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_nested_stre__nto_long_names_hashid as - varchar -), '') || '-' || coalesce(cast(double_array_data as - varchar -), '') || '-' || coalesce(cast("DATA" as - varchar -), '') || '-' || coalesce(cast("column`_'with""_quotes" as - varchar -), '') as - varchar -)) as _airbyte_partition_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab2" tmp --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql deleted file mode 100644 index 7257a111900c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_partition_hashid, - jsonb_extract_path_text(_airbyte_nested_data, 'currency') as currency, - _airbyte_emitted_at -from "postgres".test_normalization."nested_stream_with_c___long_names_partition" as table_alias -cross join jsonb_array_elements( - case jsonb_typeof("DATA") - when 'array' then "DATA" - else '[]' end - ) as _airbyte_nested_data -where "DATA" is not null --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql deleted file mode 100644 index 190e0bc7efdc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as - varchar -) as currency, - _airbyte_emitted_at -from "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab1" --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql deleted file mode 100644 index 4d934eb2ca2e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_partition_hashid as - varchar -), '') || '-' || coalesce(cast(currency as - varchar -), '') as - varchar -)) as _airbyte_data_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab2" tmp --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql deleted file mode 100644 index 4f5262e747d7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_partition_hashid, - jsonb_extract_path_text(_airbyte_nested_data, 'currency') as currency, - _airbyte_emitted_at -from "postgres".test_normalization."nested_stream_with_c___long_names_partition" as table_alias -cross join jsonb_array_elements( - case jsonb_typeof("column`_'with""_quotes") - when 'array' then "column`_'with""_quotes" - else '[]' end - ) as _airbyte_nested_data -where "column`_'with""_quotes" is not null --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql deleted file mode 100644 index 6731a64c8070..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as - varchar -) as currency, - _airbyte_emitted_at -from "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab1" --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql deleted file mode 100644 index 46bb87ddebc0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_partition_hashid as - varchar -), '') || '-' || coalesce(cast(currency as - varchar -), '') as - varchar -)) as _airbyte_column___with__quotes_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab2" tmp --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql deleted file mode 100644 index e6cab758b3bd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_partition_hashid, - jsonb_extract_path_text(_airbyte_nested_data, 'id') as "id", - _airbyte_emitted_at -from "postgres".test_normalization."nested_stream_with_c___long_names_partition" as table_alias -cross join jsonb_array_elements( - case jsonb_typeof(double_array_data) - when 'array' then double_array_data - else '[]' end - ) as _airbyte_nested_data -where double_array_data is not null --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql deleted file mode 100644 index dab5ccb08a17..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast("id" as - varchar -) as "id", - _airbyte_emitted_at -from "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab1" --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql deleted file mode 100644 index 161af77ca437..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_partition_hashid as - varchar -), '') || '-' || coalesce(cast("id" as - varchar -), '') as - varchar -)) as _airbyte_double_array_data_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab2" tmp --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql deleted file mode 100644 index 0cc444ec6bf5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path_text(_airbyte_data, 'date') as "date", - - jsonb_extract_path(table_alias._airbyte_data, 'partition') - as "partition", - _airbyte_emitted_at -from "postgres".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias --- nested_stream_with_c__lting_into_long_names - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql deleted file mode 100644 index 90dadf5dd098..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - varchar -) as "id", - cast("date" as - varchar -) as "date", - cast("partition" as - jsonb -) as "partition", - _airbyte_emitted_at -from "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab1" --- nested_stream_with_c__lting_into_long_names - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql deleted file mode 100644 index b7e5ed3ee3dc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast("date" as - varchar -), '') || '-' || coalesce(cast("partition" as - varchar -), '') as - varchar -)) as _airbyte_nested_stre__nto_long_names_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab2" tmp --- nested_stream_with_c__lting_into_long_names - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab4.sql deleted file mode 100644 index 6748eb12b538..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab4.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab4__dbt_tmp" as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_nested_stre__nto_long_names_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab3" tmp --- nested_stream_with_c__lting_into_long_names from "postgres".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql deleted file mode 100644 index 27fd33e91cea..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path_text(_airbyte_data, 'date') as "date", - _airbyte_emitted_at -from "postgres".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias --- non_nested_stream_wi__lting_into_long_names - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql deleted file mode 100644 index 648f2c41da70..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql +++ /dev/null @@ -1,15 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - varchar -) as "id", - cast("date" as - varchar -) as "date", - _airbyte_emitted_at -from "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab1" --- non_nested_stream_wi__lting_into_long_names - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql deleted file mode 100644 index 40ba8b06e03d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast("date" as - varchar -), '') as - varchar -)) as _airbyte_non_nested___nto_long_names_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab2" tmp --- non_nested_stream_wi__lting_into_long_names - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab1.sql deleted file mode 100644 index 018a584a1ac0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."unnest_alias_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path(_airbyte_data, 'children') as children, - _airbyte_emitted_at -from "postgres".test_normalization._airbyte_raw_unnest_alias as table_alias --- unnest_alias - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab2.sql deleted file mode 100644 index 6e420b273a89..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."unnest_alias_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - bigint -) as "id", - children, - _airbyte_emitted_at -from "postgres"._airbyte_test_normalization."unnest_alias_ab1" --- unnest_alias - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab3.sql deleted file mode 100644 index 182ddc64ac42..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."unnest_alias_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast(children as - varchar -), '') as - varchar -)) as _airbyte_unnest_alias_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."unnest_alias_ab2" tmp --- unnest_alias - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql deleted file mode 100644 index c14e14b49980..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ /dev/null @@ -1,21 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."unnest_alias_children_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_unnest_alias_hashid, - jsonb_extract_path_text(_airbyte_nested_data, 'ab_id') as ab_id, - - jsonb_extract_path(_airbyte_nested_data, 'owner') - as "owner", - _airbyte_emitted_at -from "postgres".test_normalization."unnest_alias" as table_alias -cross join jsonb_array_elements( - case jsonb_typeof(children) - when 'array' then children - else '[]' end - ) as _airbyte_nested_data -where children is not null --- children at unnest_alias/children - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql deleted file mode 100644 index 75278719675d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."unnest_alias_children_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_unnest_alias_hashid, - cast(ab_id as - bigint -) as ab_id, - cast("owner" as - jsonb -) as "owner", - _airbyte_emitted_at -from "postgres"._airbyte_test_normalization."unnest_alias_children_ab1" --- children at unnest_alias/children - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql deleted file mode 100644 index 8aa509953c79..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."unnest_alias_children_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_unnest_alias_hashid as - varchar -), '') || '-' || coalesce(cast(ab_id as - varchar -), '') || '-' || coalesce(cast("owner" as - varchar -), '') as - varchar -)) as _airbyte_children_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."unnest_alias_children_ab2" tmp --- children at unnest_alias/children - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql deleted file mode 100644 index 1d3ff52740e1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ /dev/null @@ -1,12 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_children_hashid, - jsonb_extract_path_text("owner", 'owner_id') as owner_id, - _airbyte_emitted_at -from "postgres".test_normalization."unnest_alias_children" as table_alias -where "owner" is not null --- owner at unnest_alias/children/owner - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql deleted file mode 100644 index 773916c973af..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_children_hashid, - cast(owner_id as - bigint -) as owner_id, - _airbyte_emitted_at -from "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab1" --- owner at unnest_alias/children/owner - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql deleted file mode 100644 index 4f2ca0385d91..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_children_hashid as - varchar -), '') || '-' || coalesce(cast(owner_id as - varchar -), '') as - varchar -)) as _airbyte_owner_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab2" tmp --- owner at unnest_alias/children/owner - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql deleted file mode 100644 index c2357389f4d2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path_text(_airbyte_data, 'date') as "date", - _airbyte_emitted_at -from "postgres".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias --- simple_stream_with_n__lting_into_long_names - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql deleted file mode 100644 index 9beefb4a2609..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql +++ /dev/null @@ -1,15 +0,0 @@ - - create view "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - varchar -) as "id", - cast("date" as - varchar -) as "date", - _airbyte_emitted_at -from "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab1" --- simple_stream_with_n__lting_into_long_names - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql deleted file mode 100644 index e34dd44daa29..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast("date" as - varchar -), '') as - varchar -)) as _airbyte_simple_stre__nto_long_names_hashid, - tmp.* -from "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab2" tmp --- simple_stream_with_n__lting_into_long_names - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql deleted file mode 100644 index 82df6d490af6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql +++ /dev/null @@ -1,14 +0,0 @@ - - - create table "postgres".test_normalization."conflict_stream_array__dbt_tmp" - as ( - --- Final base SQL model -select - "id", - conflict_stream_array, - _airbyte_emitted_at, - _airbyte_conflict_stream_array_hashid -from "postgres"._airbyte_test_normalization."conflict_stream_array_ab3" --- conflict_stream_array from "postgres".test_normalization._airbyte_raw_conflict_stream_array - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql deleted file mode 100644 index 2621ce2ba6eb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql +++ /dev/null @@ -1,14 +0,0 @@ - - - create table "postgres".test_normalization."conflict_stream_name__dbt_tmp" - as ( - --- Final base SQL model -select - "id", - conflict_stream_name, - _airbyte_emitted_at, - _airbyte_conflict_stream_name_hashid -from "postgres"._airbyte_test_normalization."conflict_stream_name_ab3" --- conflict_stream_name from "postgres".test_normalization._airbyte_raw_conflict_stream_name - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql deleted file mode 100644 index 42f7228477cb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql +++ /dev/null @@ -1,14 +0,0 @@ - - - create table "postgres".test_normalization."conflict_stream_name___conflict_stream_name__dbt_tmp" - as ( - --- Final base SQL model -select - _airbyte_conflict_stream_name_2_hashid, - groups, - _airbyte_emitted_at, - _airbyte_conflict_stream_name_3_hashid -from "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab3" --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from "postgres".test_normalization."conflict_stream_name_conflict_stream_name" - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql deleted file mode 100644 index d78bcb97cd2a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ /dev/null @@ -1,14 +0,0 @@ - - - create table "postgres".test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp" - as ( - --- Final base SQL model -select - _airbyte_conflict_stream_name_hashid, - conflict_stream_name, - _airbyte_emitted_at, - _airbyte_conflict_stream_name_2_hashid -from "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab3" --- conflict_stream_name at conflict_stream_name/conflict_stream_name from "postgres".test_normalization."conflict_stream_name" - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql deleted file mode 100644 index a1fa74a3691d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ /dev/null @@ -1,14 +0,0 @@ - - - create table "postgres".test_normalization."conflict_stream_scalar__dbt_tmp" - as ( - --- Final base SQL model -select - "id", - conflict_stream_scalar, - _airbyte_emitted_at, - _airbyte_conflict_stream_scalar_hashid -from "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab3" --- conflict_stream_scalar from "postgres".test_normalization._airbyte_raw_conflict_stream_scalar - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql deleted file mode 100644 index 05b5585f6de3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql +++ /dev/null @@ -1,16 +0,0 @@ - - - create table "postgres".test_normalization."nested_stream_with_c___long_names_partition__dbt_tmp" - as ( - --- Final base SQL model -select - _airbyte_nested_stre__nto_long_names_hashid, - double_array_data, - "DATA", - "column`_'with""_quotes", - _airbyte_emitted_at, - _airbyte_partition_hashid -from "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab3" --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "postgres".test_normalization."nested_stream_with_c__lting_into_long_names" - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql deleted file mode 100644 index 8a5b484f27c8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql +++ /dev/null @@ -1,14 +0,0 @@ - - - create table "postgres".test_normalization."nested_stream_with_c___names_partition_data__dbt_tmp" - as ( - --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_emitted_at, - _airbyte_data_hashid -from "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab3" --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "postgres".test_normalization."nested_stream_with_c___long_names_partition" - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql deleted file mode 100644 index 357223729efc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql +++ /dev/null @@ -1,14 +0,0 @@ - - - create table "postgres".test_normalization."nested_stream_with_c__column___with__quotes__dbt_tmp" - as ( - --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_emitted_at, - _airbyte_column___with__quotes_hashid -from "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab3" --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from "postgres".test_normalization."nested_stream_with_c___long_names_partition" - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql deleted file mode 100644 index a1ee4e50790b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql +++ /dev/null @@ -1,14 +0,0 @@ - - - create table "postgres".test_normalization."nested_stream_with_c__ion_double_array_data__dbt_tmp" - as ( - --- Final base SQL model -select - _airbyte_partition_hashid, - "id", - _airbyte_emitted_at, - _airbyte_double_array_data_hashid -from "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab3" --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "postgres".test_normalization."nested_stream_with_c___long_names_partition" - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c__lting_into_long_names.sql deleted file mode 100644 index cf0e6ec3c444..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c__lting_into_long_names.sql +++ /dev/null @@ -1,16 +0,0 @@ - - - create table "postgres".test_normalization."nested_stream_with_c__lting_into_long_names__dbt_tmp" - as ( - --- Final base SQL model -select - "id", - "date", - "partition", - _airbyte_emitted_at, - _airbyte_nested_stre__nto_long_names_hashid -from "postgres".test_normalization."nested_stream_with_c__lting_into_long_names_scd" --- nested_stream_with_c__lting_into_long_names from "postgres".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names -where _airbyte_active_row = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql deleted file mode 100644 index 8601af054b1a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql +++ /dev/null @@ -1,25 +0,0 @@ - - - create table "postgres".test_normalization."nested_stream_with_c__lting_into_long_names_scd__dbt_tmp" - as ( - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - "id", - "date", - "partition", - "date" as _airbyte_start_at, - lag("date") over ( - partition by "id" - order by "date" is null asc, "date" desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag("date") over ( - partition by "id" - order by "date" is null asc, "date" desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_nested_stre__nto_long_names_hashid -from "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab4" --- nested_stream_with_c__lting_into_long_names from "postgres".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names -where _airbyte_row_num = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql deleted file mode 100644 index f3d8553a5893..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql +++ /dev/null @@ -1,14 +0,0 @@ - - - create table "postgres".test_normalization."non_nested_stream_wi__lting_into_long_names__dbt_tmp" - as ( - --- Final base SQL model -select - "id", - "date", - _airbyte_emitted_at, - _airbyte_non_nested___nto_long_names_hashid -from "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab3" --- non_nested_stream_wi__lting_into_long_names from "postgres".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql deleted file mode 100644 index 546d7d6db6a1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,14 +0,0 @@ - - - create table "postgres".test_normalization."unnest_alias__dbt_tmp" - as ( - --- Final base SQL model -select - "id", - children, - _airbyte_emitted_at, - _airbyte_unnest_alias_hashid -from "postgres"._airbyte_test_normalization."unnest_alias_ab3" --- unnest_alias from "postgres".test_normalization._airbyte_raw_unnest_alias - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql deleted file mode 100644 index 36d780ad4cac..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create table "postgres".test_normalization."unnest_alias_children__dbt_tmp" - as ( - --- Final base SQL model -select - _airbyte_unnest_alias_hashid, - ab_id, - "owner", - _airbyte_emitted_at, - _airbyte_children_hashid -from "postgres"._airbyte_test_normalization."unnest_alias_children_ab3" --- children at unnest_alias/children from "postgres".test_normalization."unnest_alias" - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql deleted file mode 100644 index 7913f47c46e9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ /dev/null @@ -1,14 +0,0 @@ - - - create table "postgres".test_normalization."unnest_alias_children_owner__dbt_tmp" - as ( - --- Final base SQL model -select - _airbyte_children_hashid, - owner_id, - _airbyte_emitted_at, - _airbyte_owner_hashid -from "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab3" --- owner at unnest_alias/children/owner from "postgres".test_normalization."unnest_alias_children" - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql deleted file mode 100644 index 87e70a69c3e6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql +++ /dev/null @@ -1,14 +0,0 @@ - - - create table "postgres".test_normalization_namespace."simple_stream_with_n__lting_into_long_names__dbt_tmp" - as ( - --- Final base SQL model -select - "id", - "date", - _airbyte_emitted_at, - _airbyte_simple_stre__nto_long_names_hashid -from "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab3" --- simple_stream_with_n__lting_into_long_names from "postgres".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql new file mode 100644 index 000000000000..ce2e7a3ec5b5 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql @@ -0,0 +1,14 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_array_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path(_airbyte_data, 'conflict_stream_array') as conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_conflict_stream_array as table_alias +-- conflict_stream_array +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql new file mode 100644 index 000000000000..a4df9ebe22b0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_array_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + varchar +) as "id", + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."conflict_stream_array_ab1" +-- conflict_stream_array +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql new file mode 100644 index 000000000000..de392f8c307b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_array_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(conflict_stream_array as + varchar +), '') as + varchar +)) as _airbyte_conflict_stream_array_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."conflict_stream_array_ab2" tmp +-- conflict_stream_array +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..da03bc0bb8db --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql @@ -0,0 +1,15 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_2_hashid, + jsonb_extract_path_text(conflict_stream_name, 'groups') as groups, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."conflict_stream_name_conflict_stream_name" as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..f40efe7753f1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_2_hashid, + cast(groups as + varchar +) as groups, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab1" +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..585ae9060bfe --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as + varchar +), '') || '-' || coalesce(cast(groups as + varchar +), '') as + varchar +)) as _airbyte_conflict_stream_name_3_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab2" tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..edec407e9552 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_name_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + + jsonb_extract_path(table_alias._airbyte_data, 'conflict_stream_name') + as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_conflict_stream_name as table_alias +-- conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..fed1290ad9a7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql @@ -0,0 +1,18 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_name_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + varchar +) as "id", + cast(conflict_stream_name as + jsonb +) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."conflict_stream_name_ab1" +-- conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..d82b4294172d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_name_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(conflict_stream_name as + varchar +), '') as + varchar +)) as _airbyte_conflict_stream_name_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."conflict_stream_name_ab2" tmp +-- conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..f0ca905a6656 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_hashid, + + jsonb_extract_path(table_alias.conflict_stream_name, 'conflict_stream_name') + as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."conflict_stream_name" as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..a3a5870bec5d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_hashid, + cast(conflict_stream_name as + jsonb +) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab1" +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..8435a2464b7b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_conflict_stream_name_hashid as + varchar +), '') || '-' || coalesce(cast(conflict_stream_name as + varchar +), '') as + varchar +)) as _airbyte_conflict_stream_name_2_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab2" tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql new file mode 100644 index 000000000000..abb9d5899986 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql @@ -0,0 +1,14 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'conflict_stream_scalar') as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_conflict_stream_scalar as table_alias +-- conflict_stream_scalar +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql new file mode 100644 index 000000000000..a5a94a9cb6c1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql @@ -0,0 +1,18 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + varchar +) as "id", + cast(conflict_stream_scalar as + bigint +) as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab1" +-- conflict_stream_scalar +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql new file mode 100644 index 000000000000..66ad40a66f43 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(conflict_stream_scalar as + varchar +), '') as + varchar +)) as _airbyte_conflict_stream_scalar_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab2" tmp +-- conflict_stream_scalar +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql new file mode 100644 index 000000000000..f6e2700418b3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_nested_stre__nto_long_names_hashid, + jsonb_extract_path("partition", 'double_array_data') as double_array_data, + jsonb_extract_path("partition", 'DATA') as "DATA", + jsonb_extract_path("partition", 'column`_''with"_quotes') as "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."nested_stream_with_c__lting_into_long_names" as table_alias +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and "partition" is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql new file mode 100644 index 000000000000..451bd82174e7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_nested_stre__nto_long_names_hashid, + double_array_data, + "DATA", + "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab1" +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql new file mode 100644 index 000000000000..eb31fad896ca --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql @@ -0,0 +1,21 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_nested_stre__nto_long_names_hashid as + varchar +), '') || '-' || coalesce(cast(double_array_data as + varchar +), '') || '-' || coalesce(cast("DATA" as + varchar +), '') || '-' || coalesce(cast("column`_'with""_quotes" as + varchar +), '') as + varchar +)) as _airbyte_partition_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab2" tmp +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql new file mode 100644 index 000000000000..602dad2547f8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql @@ -0,0 +1,21 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + jsonb_extract_path_text(_airbyte_nested_data, 'currency') as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."nested_stream_with_c___long_names_partition" as table_alias +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +cross join jsonb_array_elements( + case jsonb_typeof("DATA") + when 'array' then "DATA" + else '[]' end + ) as _airbyte_nested_data +where 1 = 1 +and "DATA" is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql new file mode 100644 index 000000000000..c51fbcb9e61f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as + varchar +) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab1" +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql new file mode 100644 index 000000000000..94e1500a8681 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_partition_hashid as + varchar +), '') || '-' || coalesce(cast(currency as + varchar +), '') as + varchar +)) as _airbyte_data_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab2" tmp +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql new file mode 100644 index 000000000000..d6e58ff5015a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql @@ -0,0 +1,21 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + jsonb_extract_path_text(_airbyte_nested_data, 'currency') as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."nested_stream_with_c___long_names_partition" as table_alias +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +cross join jsonb_array_elements( + case jsonb_typeof("column`_'with""_quotes") + when 'array' then "column`_'with""_quotes" + else '[]' end + ) as _airbyte_nested_data +where 1 = 1 +and "column`_'with""_quotes" is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql new file mode 100644 index 000000000000..3517ce920c9a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as + varchar +) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab1" +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql new file mode 100644 index 000000000000..37d739b12c2c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_partition_hashid as + varchar +), '') || '-' || coalesce(cast(currency as + varchar +), '') as + varchar +)) as _airbyte_column___with__quotes_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab2" tmp +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql new file mode 100644 index 000000000000..e18ebddd1a8c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql @@ -0,0 +1,21 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + jsonb_extract_path_text(_airbyte_nested_data, 'id') as "id", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."nested_stream_with_c___long_names_partition" as table_alias +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +cross join jsonb_array_elements( + case jsonb_typeof(double_array_data) + when 'array' then double_array_data + else '[]' end + ) as _airbyte_nested_data +where 1 = 1 +and double_array_data is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql new file mode 100644 index 000000000000..d716b1eb2f5d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast("id" as + varchar +) as "id", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab1" +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql new file mode 100644 index 000000000000..f4511754ecfc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_partition_hashid as + varchar +), '') || '-' || coalesce(cast("id" as + varchar +), '') as + varchar +)) as _airbyte_double_array_data_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab2" tmp +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql new file mode 100644 index 000000000000..fd8bdeb88d0c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql @@ -0,0 +1,18 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + + jsonb_extract_path(table_alias._airbyte_data, 'partition') + as "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias +-- nested_stream_with_c__lting_into_long_names +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql new file mode 100644 index 000000000000..3eacd30f282f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql @@ -0,0 +1,22 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + varchar +) as "id", + cast("date" as + varchar +) as "date", + cast("partition" as + jsonb +) as "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab1" +-- nested_stream_with_c__lting_into_long_names +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql new file mode 100644 index 000000000000..2a1e4ca28f20 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql @@ -0,0 +1,14 @@ + + create view "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias +-- non_nested_stream_wi__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql new file mode 100644 index 000000000000..51620aee9290 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql @@ -0,0 +1,18 @@ + + create view "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + varchar +) as "id", + cast("date" as + varchar +) as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab1" +-- non_nested_stream_wi__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql new file mode 100644 index 000000000000..151cdb6e10a3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') as + varchar +)) as _airbyte_non_nested___nto_long_names_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab2" tmp +-- non_nested_stream_wi__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql new file mode 100644 index 000000000000..b70f91add752 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql @@ -0,0 +1,15 @@ + + create view "postgres"._airbyte_test_normalization."unnest_alias_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path(_airbyte_data, 'children') as children, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_unnest_alias as table_alias +-- unnest_alias +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql new file mode 100644 index 000000000000..86b9fb2c93f3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."unnest_alias_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + children, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."unnest_alias_ab1" +-- unnest_alias +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql new file mode 100644 index 000000000000..f23e4c1aca53 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql @@ -0,0 +1,18 @@ + + create view "postgres"._airbyte_test_normalization."unnest_alias_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(children as + varchar +), '') as + varchar +)) as _airbyte_unnest_alias_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."unnest_alias_ab2" tmp +-- unnest_alias +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql new file mode 100644 index 000000000000..04ce3aa2eacc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql @@ -0,0 +1,24 @@ + + create view "postgres"._airbyte_test_normalization."unnest_alias_children_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_unnest_alias_hashid, + jsonb_extract_path_text(_airbyte_nested_data, 'ab_id') as ab_id, + + jsonb_extract_path(_airbyte_nested_data, 'owner') + as "owner", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."unnest_alias" as table_alias +-- children at unnest_alias/children +cross join jsonb_array_elements( + case jsonb_typeof(children) + when 'array' then children + else '[]' end + ) as _airbyte_nested_data +where 1 = 1 +and children is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql new file mode 100644 index 000000000000..00521425f33e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql @@ -0,0 +1,19 @@ + + create view "postgres"._airbyte_test_normalization."unnest_alias_children_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_unnest_alias_hashid, + cast(ab_id as + bigint +) as ab_id, + cast("owner" as + jsonb +) as "owner", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."unnest_alias_children_ab1" +-- children at unnest_alias/children +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql new file mode 100644 index 000000000000..f11f03175e3b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql @@ -0,0 +1,19 @@ + + create view "postgres"._airbyte_test_normalization."unnest_alias_children_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_unnest_alias_hashid as + varchar +), '') || '-' || coalesce(cast(ab_id as + varchar +), '') || '-' || coalesce(cast("owner" as + varchar +), '') as + varchar +)) as _airbyte_children_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."unnest_alias_children_ab2" tmp +-- children at unnest_alias/children +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql new file mode 100644 index 000000000000..3aafcf647131 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql @@ -0,0 +1,15 @@ + + create view "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_children_hashid, + jsonb_extract_path_text("owner", 'owner_id') as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."unnest_alias_children" as table_alias +-- owner at unnest_alias/children/owner +where 1 = 1 +and "owner" is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql new file mode 100644 index 000000000000..b2ca5615ca90 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_children_hashid, + cast(owner_id as + bigint +) as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab1" +-- owner at unnest_alias/children/owner +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql new file mode 100644 index 000000000000..8239aba10289 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_children_hashid as + varchar +), '') || '-' || coalesce(cast(owner_id as + varchar +), '') as + varchar +)) as _airbyte_owner_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab2" tmp +-- owner at unnest_alias/children/owner +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql new file mode 100644 index 000000000000..5211a6eeff16 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql @@ -0,0 +1,14 @@ + + create view "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias +-- simple_stream_with_n__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql new file mode 100644 index 000000000000..8f67edcf78c3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql @@ -0,0 +1,18 @@ + + create view "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + varchar +) as "id", + cast("date" as + varchar +) as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab1" +-- simple_stream_with_n__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql new file mode 100644 index 000000000000..1c204c3383bb --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') as + varchar +)) as _airbyte_simple_stre__nto_long_names_hashid, + tmp.* +from "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab2" tmp +-- simple_stream_with_n__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql new file mode 100644 index 000000000000..d844645b4961 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql @@ -0,0 +1,81 @@ + + + + create table "postgres".test_normalization."nested_stream_with_c__lting_into_long_names_scd" + as ( + +with + +input_data as ( + select * + from "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab3" + -- nested_stream_with_c__lting_into_long_names from "postgres".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast("id" as + varchar +), '') as + varchar +)) as _airbyte_unique_key, + "id", + "date", + "partition", + "date" as _airbyte_start_at, + lag("date") over ( + partition by "id" + order by + "date" is null asc, + "date" desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag("date") over ( + partition by "id" + order by + "date" is null asc, + "date" desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_nested_stre__nto_long_names_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(coalesce(cast(_airbyte_unique_key as + varchar +), '') || '-' || coalesce(cast(_airbyte_start_at as + varchar +), '') || '-' || coalesce(cast(_airbyte_emitted_at as + varchar +), '') as + varchar +)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + "id", + "date", + "partition", + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_nested_stre__nto_long_names_hashid +from dedup_data where _airbyte_row_num = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names.sql new file mode 100644 index 000000000000..a90caa4b0cb6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names.sql @@ -0,0 +1,23 @@ + + + + create table "postgres".test_normalization."nested_stream_with_c__lting_into_long_names" + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + "id", + "date", + "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_nested_stre__nto_long_names_hashid +from "postgres".test_normalization."nested_stream_with_c__lting_into_long_names_scd" +-- nested_stream_with_c__lting_into_long_names from "postgres".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 +and _airbyte_active_row = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..b1be69446486 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql @@ -0,0 +1,20 @@ + + + + create table "postgres".test_normalization."unnest_alias" + as ( + +-- Final base SQL model +select + "id", + children, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_unnest_alias_hashid +from "postgres"._airbyte_test_normalization."unnest_alias_ab3" +-- unnest_alias from "postgres".test_normalization._airbyte_raw_unnest_alias +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql new file mode 100644 index 000000000000..24d04f946c89 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."conflict_stream_array__dbt_tmp" + as ( + +-- Final base SQL model +select + "id", + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_conflict_stream_array_hashid +from "postgres"._airbyte_test_normalization."conflict_stream_array_ab3" +-- conflict_stream_array from "postgres".test_normalization._airbyte_raw_conflict_stream_array +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql new file mode 100644 index 000000000000..a664bb56d781 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."conflict_stream_name__dbt_tmp" + as ( + +-- Final base SQL model +select + "id", + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_hashid +from "postgres"._airbyte_test_normalization."conflict_stream_name_ab3" +-- conflict_stream_name from "postgres".test_normalization._airbyte_raw_conflict_stream_name +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql new file mode 100644 index 000000000000..0e06a3243fa2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."conflict_stream_name___conflict_stream_name__dbt_tmp" + as ( + +-- Final base SQL model +select + _airbyte_conflict_stream_name_2_hashid, + groups, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_3_hashid +from "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab3" +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from "postgres".test_normalization."conflict_stream_name_conflict_stream_name" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql new file mode 100644 index 000000000000..df6e327e1620 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp" + as ( + +-- Final base SQL model +select + _airbyte_conflict_stream_name_hashid, + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_2_hashid +from "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab3" +-- conflict_stream_name at conflict_stream_name/conflict_stream_name from "postgres".test_normalization."conflict_stream_name" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql new file mode 100644 index 000000000000..bdcd7a2664c1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."conflict_stream_scalar__dbt_tmp" + as ( + +-- Final base SQL model +select + "id", + conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_conflict_stream_scalar_hashid +from "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab3" +-- conflict_stream_scalar from "postgres".test_normalization._airbyte_raw_conflict_stream_scalar +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql new file mode 100644 index 000000000000..a9781fcf92d2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql @@ -0,0 +1,19 @@ + + + create table "postgres".test_normalization."nested_stream_with_c___long_names_partition__dbt_tmp" + as ( + +-- Final base SQL model +select + _airbyte_nested_stre__nto_long_names_hashid, + double_array_data, + "DATA", + "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_partition_hashid +from "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab3" +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "postgres".test_normalization."nested_stream_with_c__lting_into_long_names" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql new file mode 100644 index 000000000000..b6c629700a04 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."nested_stream_with_c___names_partition_data__dbt_tmp" + as ( + +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_data_hashid +from "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab3" +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "postgres".test_normalization."nested_stream_with_c___long_names_partition" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql new file mode 100644 index 000000000000..037c53598daf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."nested_stream_with_c__column___with__quotes__dbt_tmp" + as ( + +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_column___with__quotes_hashid +from "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab3" +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from "postgres".test_normalization."nested_stream_with_c___long_names_partition" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql new file mode 100644 index 000000000000..6d3ed538c39d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."nested_stream_with_c__ion_double_array_data__dbt_tmp" + as ( + +-- Final base SQL model +select + _airbyte_partition_hashid, + "id", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_double_array_data_hashid +from "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab3" +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "postgres".test_normalization."nested_stream_with_c___long_names_partition" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql new file mode 100644 index 000000000000..0939d42acba7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."non_nested_stream_wi__lting_into_long_names__dbt_tmp" + as ( + +-- Final base SQL model +select + "id", + "date", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_non_nested___nto_long_names_hashid +from "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab3" +-- non_nested_stream_wi__lting_into_long_names from "postgres".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql new file mode 100644 index 000000000000..5248364bc8cd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -0,0 +1,18 @@ + + + create table "postgres".test_normalization."unnest_alias_children__dbt_tmp" + as ( + +-- Final base SQL model +select + _airbyte_unnest_alias_hashid, + ab_id, + "owner", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_children_hashid +from "postgres"._airbyte_test_normalization."unnest_alias_children_ab3" +-- children at unnest_alias/children from "postgres".test_normalization."unnest_alias" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql new file mode 100644 index 000000000000..b3074c01b9c9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."unnest_alias_children_owner__dbt_tmp" + as ( + +-- Final base SQL model +select + _airbyte_children_hashid, + owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_owner_hashid +from "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab3" +-- owner at unnest_alias/children/owner from "postgres".test_normalization."unnest_alias_children" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql new file mode 100644 index 000000000000..ed8e802bc153 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization_namespace."simple_stream_with_n__lting_into_long_names__dbt_tmp" + as ( + +-- Final base SQL model +select + "id", + "date", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_simple_stre__nto_long_names_hashid +from "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab3" +-- simple_stream_with_n__lting_into_long_names from "postgres".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql new file mode 100644 index 000000000000..0d5dd2d7fcac --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql @@ -0,0 +1,20 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') || '-' || coalesce(cast("partition" as + varchar +), '') as + varchar +)) as _airbyte_nested_stre__nto_long_names_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab2" tmp +-- nested_stream_with_c__lting_into_long_names +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql deleted file mode 100644 index 62d0ec8cc61b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql +++ /dev/null @@ -1,19 +0,0 @@ -with table_row_counts as ( - select distinct count(*) as row_count, 2 as expected_count - from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} -union all - select distinct count(*) as row_count, 2 as expected_count - from {{ ref('nested_stream_with_c__lting_into_long_names') }} -union all - select distinct count(*) as row_count, 2 as expected_count - from {{ ref('nested_stream_with_c___long_names_partition') }} -union all - select count(distinct currency) as row_count, 1 as expected_count - from {{ ref('nested_stream_with_c___names_partition_data') }} --- union all --- select count(distinct id) as row_count, 3 as expected_count --- from {{ ref('nested_stream_with_c__ion_double_array_data') }} -) -select * -from table_row_counts -where row_count != expected_count diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql index bcc29f442b1c..b6d1d437f60a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as {{ adapter.quote('id') }}, {{ json_extract_array('_airbyte_data', ['conflict_stream_array'], ['conflict_stream_array']) }} as conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} as table_alias -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql index 2f109e54445b..44a922d69754 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast({{ adapter.quote('id') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('id') }}, conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_array_ab1') }} -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql index 5422c2c86745..c2ab9a61b17c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('conflict_stream_array_ab2') }} tmp -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql index 77e7bd31c512..08d114e6ce78 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql @@ -1,10 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_conflict_stream_name_2_hashid, {{ json_extract_scalar('conflict_stream_name', ['groups'], ['groups']) }} as groups, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_conflict_stream_name') }} as table_alias -where conflict_stream_name is not null -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql index 683cf66727ac..8179e5d3f565 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_conflict_stream_name_2_hashid, cast(groups as {{ dbt_utils.type_string() }}) as groups, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name___conflict_stream_name_ab1') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql index 25a48098b5cf..91728d4b7441 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('conflict_stream_name___conflict_stream_name_ab2') }} tmp -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql index 1026b2100308..52ad1cb15400 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as {{ adapter.quote('id') }}, {{ json_extract('table_alias', '_airbyte_data', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} as table_alias -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql index 020604d3bd83..19af2a871603 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast({{ adapter.quote('id') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('id') }}, cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_ab1') }} -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql index 9d6d6c50ccf9..3c048edff728 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('conflict_stream_name_ab2') }} tmp -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql index d6e2edb75ced..62d7cc9c812b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql @@ -1,10 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_conflict_stream_name_hashid, {{ json_extract('table_alias', 'conflict_stream_name', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name') }} as table_alias -where conflict_stream_name is not null -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql index 463a878933f8..5d564afc1564 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_conflict_stream_name_hashid, cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_conflict_stream_name_ab1') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql index f8d96123ce25..3e8ebb5d34d3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('conflict_stream_name_conflict_stream_name_ab2') }} tmp -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql index 95f08b861d80..87f7dae0e73e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as {{ adapter.quote('id') }}, {{ json_extract_scalar('_airbyte_data', ['conflict_stream_scalar'], ['conflict_stream_scalar']) }} as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} as table_alias -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql index cd0a70325db6..1f254edd07e1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast({{ adapter.quote('id') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('id') }}, cast(conflict_stream_scalar as {{ dbt_utils.type_bigint() }}) as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_scalar_ab1') }} -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql index bafebc8d9113..8a1fb1abcdb9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('conflict_stream_scalar_ab2') }} tmp -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql index 11d7c57a019f..c23eb81cdb1f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql @@ -1,12 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_nested_stre__nto_long_names_hashid, {{ json_extract_array(adapter.quote('partition'), ['double_array_data'], ['double_array_data']) }} as double_array_data, {{ json_extract_array(adapter.quote('partition'), ['DATA'], ['DATA']) }} as {{ adapter.quote('DATA') }}, {{ json_extract_array(adapter.quote('partition'), ['column`_\'with"_quotes'], ['column___with__quotes']) }} as {{ adapter.quote('column`_\'with""_quotes') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_c__lting_into_long_names') }} as table_alias -where {{ adapter.quote('partition') }} is not null -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and {{ adapter.quote('partition') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql index d334fa9e3af9..de62103f6c01 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql @@ -1,11 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_nested_stre__nto_long_names_hashid, double_array_data, {{ adapter.quote('DATA') }}, {{ adapter.quote('column`_\'with""_quotes') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_c___long_names_partition_ab1') }} -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql index a6fe0d5760ef..03b4ba44eaa8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -10,4 +15,5 @@ select tmp.* from {{ ref('nested_stream_with_c___long_names_partition_ab2') }} tmp -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql index 871d33d96ae1..5c39f919bd0c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql @@ -1,12 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_c___long_names_partition', 'partition', adapter.quote('DATA')) }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value(adapter.quote('DATA')), ['currency'], ['currency']) }} as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_c___long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', adapter.quote('DATA')) }} -where {{ adapter.quote('DATA') }} is not null -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +{{ cross_join_unnest('partition', adapter.quote('DATA')) }} +where 1 = 1 +and {{ adapter.quote('DATA') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql index 4034a1495a09..670fef499d16 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(currency as {{ dbt_utils.type_string() }}) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_c___names_partition_data_ab1') }} -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql index 4e30a677a9e7..962f839574c5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('nested_stream_with_c___names_partition_data_ab2') }} tmp -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql index 31ac76876f60..7445f62f1469 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql @@ -1,12 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_c___long_names_partition', 'partition', adapter.quote('column`_\'with""_quotes')) }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value(adapter.quote('column`_\'with""_quotes')), ['currency'], ['currency']) }} as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_c___long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', adapter.quote('column`_\'with""_quotes')) }} -where {{ adapter.quote('column`_\'with""_quotes') }} is not null -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +{{ cross_join_unnest('partition', adapter.quote('column`_\'with""_quotes')) }} +where 1 = 1 +and {{ adapter.quote('column`_\'with""_quotes') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql index 1f1d43c09a26..94780e7b4e93 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(currency as {{ dbt_utils.type_string() }}) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_c__column___with__quotes_ab1') }} -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql index d6ea901a5e6f..8bf1b1d6c988 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('nested_stream_with_c__column___with__quotes_ab2') }} tmp -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql index 83232f67277e..d5254e59dd9e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql @@ -1,12 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_c___long_names_partition', 'partition', 'double_array_data') }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value('double_array_data'), ['id'], ['id']) }} as {{ adapter.quote('id') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_c___long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', 'double_array_data') }} -where double_array_data is not null -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +{{ cross_join_unnest('partition', 'double_array_data') }} +where 1 = 1 +and double_array_data is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql index bcaa8b5c53d5..0f5f7ae3a86e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast({{ adapter.quote('id') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('id') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_c__ion_double_array_data_ab1') }} -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql index 351d652df557..7231c119bc08 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('nested_stream_with_c__ion_double_array_data_ab2') }} tmp -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql index 5888a043ab45..6d69fcd436cf 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql @@ -1,10 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as {{ adapter.quote('id') }}, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, {{ json_extract('table_alias', '_airbyte_data', ['partition'], ['partition']) }} as {{ adapter.quote('partition') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} as table_alias -- nested_stream_with_c__lting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql index 11d64cbafcba..a257c6a59026 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql @@ -1,10 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast({{ adapter.quote('id') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('id') }}, cast({{ adapter.quote('date') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('date') }}, cast({{ adapter.quote('partition') }} as {{ type_json() }}) as {{ adapter.quote('partition') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_c__lting_into_long_names_ab1') }} -- nested_stream_with_c__lting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql deleted file mode 100644 index 658f253418f7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql +++ /dev/null @@ -1,12 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - adapter.quote('id'), - adapter.quote('date'), - adapter.quote('partition'), - ]) }} as _airbyte_nested_stre__nto_long_names_hashid, - tmp.* -from {{ ref('nested_stream_with_c__lting_into_long_names_ab2') }} tmp --- nested_stream_with_c__lting_into_long_names - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab4.sql deleted file mode 100644 index 6a964a409427..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_nested_stre__nto_long_names_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('nested_stream_with_c__lting_into_long_names_ab3') }} tmp --- nested_stream_with_c__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql index b844098ba9c0..9f67e97c0ad1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as {{ adapter.quote('id') }}, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names') }} as table_alias -- non_nested_stream_wi__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql index 5b3a3574bf40..054b91557303 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast({{ adapter.quote('id') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('id') }}, cast({{ adapter.quote('date') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('date') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('non_nested_stream_wi__lting_into_long_names_ab1') }} -- non_nested_stream_wi__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql index c8e27a19a68d..1c7fb942b403 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('non_nested_stream_wi__lting_into_long_names_ab2') }} tmp -- non_nested_stream_wi__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql index a0e70f6b55be..fbf3b9b1c3a2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as {{ adapter.quote('id') }}, {{ json_extract_array('_airbyte_data', ['children'], ['children']) }} as children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} as table_alias -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql index 6fb7cfe48e7b..0af0456e3965 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast({{ adapter.quote('id') }} as {{ dbt_utils.type_bigint() }}) as {{ adapter.quote('id') }}, children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_ab1') }} -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql index 340b65b8eeaa..39fdd94517f6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,6 @@ select tmp.* from {{ ref('unnest_alias_ab2') }} tmp -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql index 2237b274e29b..45a193100da8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql @@ -1,13 +1,21 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('unnest_alias', 'unnest_alias', 'children') }} select _airbyte_unnest_alias_hashid, {{ json_extract_scalar(unnested_column_value('children'), ['ab_id'], ['ab_id']) }} as ab_id, {{ json_extract('', unnested_column_value('children'), ['owner'], ['owner']) }} as {{ adapter.quote('owner') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias') }} as table_alias -{{ cross_join_unnest('unnest_alias', 'children') }} -where children is not null -- children at unnest_alias/children +{{ cross_join_unnest('unnest_alias', 'children') }} +where 1 = 1 +and children is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql index 435447589db2..b93715ce6a95 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql @@ -1,10 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_unnest_alias_hashid, cast(ab_id as {{ dbt_utils.type_bigint() }}) as ab_id, cast({{ adapter.quote('owner') }} as {{ type_json() }}) as {{ adapter.quote('owner') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children_ab1') }} -- children at unnest_alias/children +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql index e1fa4b470cc8..5080e4179aef 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -9,4 +14,5 @@ select tmp.* from {{ ref('unnest_alias_children_ab2') }} tmp -- children at unnest_alias/children +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql index 9085180f4824..6ae9fd7cf4ac 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql @@ -1,10 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_children_hashid, {{ json_extract_scalar(adapter.quote('owner'), ['owner_id'], ['owner_id']) }} as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children') }} as table_alias -where {{ adapter.quote('owner') }} is not null -- owner at unnest_alias/children/owner +where 1 = 1 +and {{ adapter.quote('owner') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql index a0f474c62db7..4ca313879bc7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_children_hashid, cast(owner_id as {{ dbt_utils.type_bigint() }}) as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children_owner_ab1') }} -- owner at unnest_alias/children/owner +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql index c5f6562c34c2..a25ec48b8a56 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('unnest_alias_children_owner_ab2') }} tmp -- owner at unnest_alias/children/owner +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql index 06a99b3588c8..b78dcd21c316 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as {{ adapter.quote('id') }}, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization_namespace', '_airbyte_raw_simple_stream_with_namespace_resulting_into_long_names') }} as table_alias -- simple_stream_with_n__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql index 70077c18dbfc..14ce22350f65 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast({{ adapter.quote('id') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('id') }}, cast({{ adapter.quote('date') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('date') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('simple_stream_with_n__lting_into_long_names_ab1') }} -- simple_stream_with_n__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql index 2d932f59e7a9..9d005b773352 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('simple_stream_with_n__lting_into_long_names_ab2') }} tmp -- simple_stream_with_n__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql new file mode 100644 index 000000000000..19fc71c02c38 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql @@ -0,0 +1,108 @@ +{{ config( + indexes = [{'columns':['_airbyte_active_row','_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('nested_stream_with_c__lting_into_long_names_ab3') }} + -- nested_stream_with_c__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('nested_stream_with_c__lting_into_long_names_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('nested_stream_with_c__lting_into_long_names_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('nested_stream_with_c__lting_into_long_names_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('nested_stream_with_c__lting_into_long_names_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('nested_stream_with_c__lting_into_long_names_ab3') }} + -- nested_stream_with_c__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + ]) }} as _airbyte_unique_key, + {{ adapter.quote('id') }}, + {{ adapter.quote('date') }}, + {{ adapter.quote('partition') }}, + {{ adapter.quote('date') }} as _airbyte_start_at, + lag({{ adapter.quote('date') }}) over ( + partition by {{ adapter.quote('id') }} + order by + {{ adapter.quote('date') }} is null asc, + {{ adapter.quote('date') }} desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag({{ adapter.quote('date') }}) over ( + partition by {{ adapter.quote('id') }} + order by + {{ adapter.quote('date') }} is null asc, + {{ adapter.quote('date') }} desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_nested_stre__nto_long_names_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + {{ adapter.quote('id') }}, + {{ adapter.quote('date') }}, + {{ adapter.quote('partition') }}, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_nested_stre__nto_long_names_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names.sql new file mode 100644 index 000000000000..9b80b49946e7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names.sql @@ -0,0 +1,22 @@ +{{ config( + indexes = [{'columns':['_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + {{ adapter.quote('id') }}, + {{ adapter.quote('date') }}, + {{ adapter.quote('partition') }}, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_nested_stre__nto_long_names_hashid +from {{ ref('nested_stream_with_c__lting_into_long_names_scd') }} +-- nested_stream_with_c__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..2dcfc8cdeb65 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql @@ -0,0 +1,19 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + {{ adapter.quote('id') }}, + children, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_unnest_alias_hashid +from {{ ref('unnest_alias_ab3') }} +-- unnest_alias from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql index d174a85b7a46..2bbf1b892334 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select {{ adapter.quote('id') }}, conflict_stream_array, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_array_hashid from {{ ref('conflict_stream_array_ab3') }} -- conflict_stream_array from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql index 87a6ace1c885..5a93704d86f2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select {{ adapter.quote('id') }}, conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_hashid from {{ ref('conflict_stream_name_ab3') }} -- conflict_stream_name from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql index 7be580e5b59a..dcf9220bfb32 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_conflict_stream_name_2_hashid, groups, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_3_hashid from {{ ref('conflict_stream_name___conflict_stream_name_ab3') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name_conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql index 94a9dc3d0139..54f7afc91440 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_conflict_stream_name_hashid, conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_2_hashid from {{ ref('conflict_stream_name_conflict_stream_name_ab3') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql index a36056903767..54c2c46282db 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select {{ adapter.quote('id') }}, conflict_stream_scalar, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_scalar_hashid from {{ ref('conflict_stream_scalar_ab3') }} -- conflict_stream_scalar from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql index 32fac08a6d40..e8cac7be4d93 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql @@ -1,12 +1,20 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_nested_stre__nto_long_names_hashid, double_array_data, {{ adapter.quote('DATA') }}, {{ adapter.quote('column`_\'with""_quotes') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_partition_hashid from {{ ref('nested_stream_with_c___long_names_partition_ab3') }} -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from {{ ref('nested_stream_with_c__lting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql index 9ba3d4c84f6e..f254f99ad20a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_data_hashid from {{ ref('nested_stream_with_c___names_partition_data_ab3') }} -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from {{ ref('nested_stream_with_c___long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql index c5af87800f26..75c2b74581e6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_column___with__quotes_hashid from {{ ref('nested_stream_with_c__column___with__quotes_ab3') }} -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from {{ ref('nested_stream_with_c___long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql index 18d2b45b021d..9311ea360679 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, {{ adapter.quote('id') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_double_array_data_hashid from {{ ref('nested_stream_with_c__ion_double_array_data_ab3') }} -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from {{ ref('nested_stream_with_c___long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__lting_into_long_names.sql deleted file mode 100644 index 6cec45f29f02..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__lting_into_long_names.sql +++ /dev/null @@ -1,12 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - {{ adapter.quote('id') }}, - {{ adapter.quote('date') }}, - {{ adapter.quote('partition') }}, - _airbyte_emitted_at, - _airbyte_nested_stre__nto_long_names_hashid -from {{ ref('nested_stream_with_c__lting_into_long_names_scd') }} --- nested_stream_with_c__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} -where _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql deleted file mode 100644 index e32972366642..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - {{ adapter.quote('id') }}, - {{ adapter.quote('date') }}, - {{ adapter.quote('partition') }}, - {{ adapter.quote('date') }} as _airbyte_start_at, - lag({{ adapter.quote('date') }}) over ( - partition by {{ adapter.quote('id') }} - order by {{ adapter.quote('date') }} is null asc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag({{ adapter.quote('date') }}) over ( - partition by {{ adapter.quote('id') }} - order by {{ adapter.quote('date') }} is null asc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_nested_stre__nto_long_names_hashid -from {{ ref('nested_stream_with_c__lting_into_long_names_ab4') }} --- nested_stream_with_c__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql index f9fe1a5fdf4e..cabe1843d8af 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select {{ adapter.quote('id') }}, {{ adapter.quote('date') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_non_nested___nto_long_names_hashid from {{ ref('non_nested_stream_wi__lting_into_long_names_ab3') }} -- non_nested_stream_wi__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql deleted file mode 100644 index cdd45120be1c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,10 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - {{ adapter.quote('id') }}, - children, - _airbyte_emitted_at, - _airbyte_unnest_alias_hashid -from {{ ref('unnest_alias_ab3') }} --- unnest_alias from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql index 9ce667c8b142..397f6ab07fb5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -1,11 +1,19 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_unnest_alias_hashid, ab_id, {{ adapter.quote('owner') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_children_hashid from {{ ref('unnest_alias_children_ab3') }} -- children at unnest_alias/children from {{ ref('unnest_alias') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql index 0ccf8e33d8d6..7aeb294ab08d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_children_hashid, owner_id, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_owner_hashid from {{ ref('unnest_alias_children_owner_ab3') }} -- owner at unnest_alias/children/owner from {{ ref('unnest_alias_children') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql index 247047c0ec69..44d939eb5a0d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization_namespace", tags=["top-level"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization_namespace", + tags = [ "top-level" ] +) }} -- Final base SQL model select {{ adapter.quote('id') }}, {{ adapter.quote('date') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_simple_stre__nto_long_names_hashid from {{ ref('simple_stream_with_n__lting_into_long_names_ab3') }} -- simple_stream_with_n__lting_into_long_names from {{ source('test_normalization_namespace', '_airbyte_raw_simple_stream_with_namespace_resulting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql new file mode 100644 index 000000000000..b0f0198eb0e3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql @@ -0,0 +1,19 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + adapter.quote('date'), + adapter.quote('partition'), + ]) }} as _airbyte_nested_stre__nto_long_names_hashid, + tmp.* +from {{ ref('nested_stream_with_c__lting_into_long_names_ab2') }} tmp +-- nested_stream_with_c__lting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql new file mode 100644 index 000000000000..ce2e7a3ec5b5 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql @@ -0,0 +1,14 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_array_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path(_airbyte_data, 'conflict_stream_array') as conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_conflict_stream_array as table_alias +-- conflict_stream_array +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql new file mode 100644 index 000000000000..a4df9ebe22b0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_array_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + varchar +) as "id", + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."conflict_stream_array_ab1" +-- conflict_stream_array +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql new file mode 100644 index 000000000000..de392f8c307b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_array_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(conflict_stream_array as + varchar +), '') as + varchar +)) as _airbyte_conflict_stream_array_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."conflict_stream_array_ab2" tmp +-- conflict_stream_array +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..da03bc0bb8db --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql @@ -0,0 +1,15 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_2_hashid, + jsonb_extract_path_text(conflict_stream_name, 'groups') as groups, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."conflict_stream_name_conflict_stream_name" as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..f40efe7753f1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_2_hashid, + cast(groups as + varchar +) as groups, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab1" +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..585ae9060bfe --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as + varchar +), '') || '-' || coalesce(cast(groups as + varchar +), '') as + varchar +)) as _airbyte_conflict_stream_name_3_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab2" tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..edec407e9552 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_name_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + + jsonb_extract_path(table_alias._airbyte_data, 'conflict_stream_name') + as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_conflict_stream_name as table_alias +-- conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..fed1290ad9a7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql @@ -0,0 +1,18 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_name_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + varchar +) as "id", + cast(conflict_stream_name as + jsonb +) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."conflict_stream_name_ab1" +-- conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..d82b4294172d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_name_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(conflict_stream_name as + varchar +), '') as + varchar +)) as _airbyte_conflict_stream_name_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."conflict_stream_name_ab2" tmp +-- conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..f0ca905a6656 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_hashid, + + jsonb_extract_path(table_alias.conflict_stream_name, 'conflict_stream_name') + as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."conflict_stream_name" as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..a3a5870bec5d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_hashid, + cast(conflict_stream_name as + jsonb +) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab1" +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..8435a2464b7b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_conflict_stream_name_hashid as + varchar +), '') || '-' || coalesce(cast(conflict_stream_name as + varchar +), '') as + varchar +)) as _airbyte_conflict_stream_name_2_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab2" tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql new file mode 100644 index 000000000000..abb9d5899986 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql @@ -0,0 +1,14 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'conflict_stream_scalar') as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_conflict_stream_scalar as table_alias +-- conflict_stream_scalar +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql new file mode 100644 index 000000000000..a5a94a9cb6c1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql @@ -0,0 +1,18 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + varchar +) as "id", + cast(conflict_stream_scalar as + bigint +) as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab1" +-- conflict_stream_scalar +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql new file mode 100644 index 000000000000..66ad40a66f43 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(conflict_stream_scalar as + varchar +), '') as + varchar +)) as _airbyte_conflict_stream_scalar_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab2" tmp +-- conflict_stream_scalar +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql new file mode 100644 index 000000000000..f6e2700418b3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_nested_stre__nto_long_names_hashid, + jsonb_extract_path("partition", 'double_array_data') as double_array_data, + jsonb_extract_path("partition", 'DATA') as "DATA", + jsonb_extract_path("partition", 'column`_''with"_quotes') as "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."nested_stream_with_c__lting_into_long_names" as table_alias +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and "partition" is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql new file mode 100644 index 000000000000..451bd82174e7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_nested_stre__nto_long_names_hashid, + double_array_data, + "DATA", + "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab1" +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql new file mode 100644 index 000000000000..eb31fad896ca --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql @@ -0,0 +1,21 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_nested_stre__nto_long_names_hashid as + varchar +), '') || '-' || coalesce(cast(double_array_data as + varchar +), '') || '-' || coalesce(cast("DATA" as + varchar +), '') || '-' || coalesce(cast("column`_'with""_quotes" as + varchar +), '') as + varchar +)) as _airbyte_partition_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab2" tmp +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql new file mode 100644 index 000000000000..602dad2547f8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql @@ -0,0 +1,21 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + jsonb_extract_path_text(_airbyte_nested_data, 'currency') as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."nested_stream_with_c___long_names_partition" as table_alias +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +cross join jsonb_array_elements( + case jsonb_typeof("DATA") + when 'array' then "DATA" + else '[]' end + ) as _airbyte_nested_data +where 1 = 1 +and "DATA" is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql new file mode 100644 index 000000000000..c51fbcb9e61f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as + varchar +) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab1" +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql new file mode 100644 index 000000000000..94e1500a8681 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_partition_hashid as + varchar +), '') || '-' || coalesce(cast(currency as + varchar +), '') as + varchar +)) as _airbyte_data_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab2" tmp +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql new file mode 100644 index 000000000000..d6e58ff5015a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql @@ -0,0 +1,21 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + jsonb_extract_path_text(_airbyte_nested_data, 'currency') as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."nested_stream_with_c___long_names_partition" as table_alias +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +cross join jsonb_array_elements( + case jsonb_typeof("column`_'with""_quotes") + when 'array' then "column`_'with""_quotes" + else '[]' end + ) as _airbyte_nested_data +where 1 = 1 +and "column`_'with""_quotes" is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql new file mode 100644 index 000000000000..3517ce920c9a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as + varchar +) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab1" +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql new file mode 100644 index 000000000000..37d739b12c2c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_partition_hashid as + varchar +), '') || '-' || coalesce(cast(currency as + varchar +), '') as + varchar +)) as _airbyte_column___with__quotes_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab2" tmp +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql new file mode 100644 index 000000000000..e18ebddd1a8c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql @@ -0,0 +1,21 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + jsonb_extract_path_text(_airbyte_nested_data, 'id') as "id", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."nested_stream_with_c___long_names_partition" as table_alias +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +cross join jsonb_array_elements( + case jsonb_typeof(double_array_data) + when 'array' then double_array_data + else '[]' end + ) as _airbyte_nested_data +where 1 = 1 +and double_array_data is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql new file mode 100644 index 000000000000..d716b1eb2f5d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast("id" as + varchar +) as "id", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab1" +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql new file mode 100644 index 000000000000..f4511754ecfc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_partition_hashid as + varchar +), '') || '-' || coalesce(cast("id" as + varchar +), '') as + varchar +)) as _airbyte_double_array_data_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab2" tmp +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql new file mode 100644 index 000000000000..fd8bdeb88d0c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql @@ -0,0 +1,18 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + + jsonb_extract_path(table_alias._airbyte_data, 'partition') + as "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias +-- nested_stream_with_c__lting_into_long_names +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql new file mode 100644 index 000000000000..3eacd30f282f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql @@ -0,0 +1,22 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + varchar +) as "id", + cast("date" as + varchar +) as "date", + cast("partition" as + jsonb +) as "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab1" +-- nested_stream_with_c__lting_into_long_names +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql new file mode 100644 index 000000000000..2a1e4ca28f20 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql @@ -0,0 +1,14 @@ + + create view "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias +-- non_nested_stream_wi__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql new file mode 100644 index 000000000000..51620aee9290 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql @@ -0,0 +1,18 @@ + + create view "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + varchar +) as "id", + cast("date" as + varchar +) as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab1" +-- non_nested_stream_wi__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql new file mode 100644 index 000000000000..151cdb6e10a3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') as + varchar +)) as _airbyte_non_nested___nto_long_names_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab2" tmp +-- non_nested_stream_wi__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql new file mode 100644 index 000000000000..b70f91add752 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql @@ -0,0 +1,15 @@ + + create view "postgres"._airbyte_test_normalization."unnest_alias_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path(_airbyte_data, 'children') as children, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_unnest_alias as table_alias +-- unnest_alias +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql new file mode 100644 index 000000000000..86b9fb2c93f3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."unnest_alias_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + children, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."unnest_alias_ab1" +-- unnest_alias +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql new file mode 100644 index 000000000000..f23e4c1aca53 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql @@ -0,0 +1,18 @@ + + create view "postgres"._airbyte_test_normalization."unnest_alias_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(children as + varchar +), '') as + varchar +)) as _airbyte_unnest_alias_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."unnest_alias_ab2" tmp +-- unnest_alias +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql new file mode 100644 index 000000000000..04ce3aa2eacc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql @@ -0,0 +1,24 @@ + + create view "postgres"._airbyte_test_normalization."unnest_alias_children_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_unnest_alias_hashid, + jsonb_extract_path_text(_airbyte_nested_data, 'ab_id') as ab_id, + + jsonb_extract_path(_airbyte_nested_data, 'owner') + as "owner", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."unnest_alias" as table_alias +-- children at unnest_alias/children +cross join jsonb_array_elements( + case jsonb_typeof(children) + when 'array' then children + else '[]' end + ) as _airbyte_nested_data +where 1 = 1 +and children is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql new file mode 100644 index 000000000000..00521425f33e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql @@ -0,0 +1,19 @@ + + create view "postgres"._airbyte_test_normalization."unnest_alias_children_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_unnest_alias_hashid, + cast(ab_id as + bigint +) as ab_id, + cast("owner" as + jsonb +) as "owner", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."unnest_alias_children_ab1" +-- children at unnest_alias/children +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql new file mode 100644 index 000000000000..f11f03175e3b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql @@ -0,0 +1,19 @@ + + create view "postgres"._airbyte_test_normalization."unnest_alias_children_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_unnest_alias_hashid as + varchar +), '') || '-' || coalesce(cast(ab_id as + varchar +), '') || '-' || coalesce(cast("owner" as + varchar +), '') as + varchar +)) as _airbyte_children_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."unnest_alias_children_ab2" tmp +-- children at unnest_alias/children +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql new file mode 100644 index 000000000000..3aafcf647131 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql @@ -0,0 +1,15 @@ + + create view "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_children_hashid, + jsonb_extract_path_text("owner", 'owner_id') as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."unnest_alias_children" as table_alias +-- owner at unnest_alias/children/owner +where 1 = 1 +and "owner" is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql new file mode 100644 index 000000000000..b2ca5615ca90 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_children_hashid, + cast(owner_id as + bigint +) as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab1" +-- owner at unnest_alias/children/owner +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql new file mode 100644 index 000000000000..8239aba10289 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_children_hashid as + varchar +), '') || '-' || coalesce(cast(owner_id as + varchar +), '') as + varchar +)) as _airbyte_owner_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab2" tmp +-- owner at unnest_alias/children/owner +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql new file mode 100644 index 000000000000..5211a6eeff16 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql @@ -0,0 +1,14 @@ + + create view "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias +-- simple_stream_with_n__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql new file mode 100644 index 000000000000..8f67edcf78c3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql @@ -0,0 +1,18 @@ + + create view "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + varchar +) as "id", + cast("date" as + varchar +) as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab1" +-- simple_stream_with_n__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql new file mode 100644 index 000000000000..1c204c3383bb --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') as + varchar +)) as _airbyte_simple_stre__nto_long_names_hashid, + tmp.* +from "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab2" tmp +-- simple_stream_with_n__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql new file mode 100644 index 000000000000..327e14609a57 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql @@ -0,0 +1,14 @@ + + delete + from "postgres".test_normalization."nested_stream_with_c__lting_into_long_names_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "nested_stream_with_c__lting_into_long_name__dbt_tmp" + ); + + insert into "postgres".test_normalization."nested_stream_with_c__lting_into_long_names_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "date", "partition", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stre__nto_long_names_hashid") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "date", "partition", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stre__nto_long_names_hashid" + from "nested_stream_with_c__lting_into_long_name__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names.sql new file mode 100644 index 000000000000..384d4a63247d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names.sql @@ -0,0 +1,14 @@ + + delete + from "postgres".test_normalization."nested_stream_with_c__lting_into_long_names" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "nested_stream_with_c__lting_into_long_name__dbt_tmp" + ); + + insert into "postgres".test_normalization."nested_stream_with_c__lting_into_long_names" ("_airbyte_unique_key", "id", "date", "partition", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stre__nto_long_names_hashid") + ( + select "_airbyte_unique_key", "id", "date", "partition", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stre__nto_long_names_hashid" + from "nested_stream_with_c__lting_into_long_name__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..7199e9f5f914 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql @@ -0,0 +1,14 @@ + + delete + from "postgres".test_normalization."unnest_alias" + where (_airbyte_ab_id) in ( + select (_airbyte_ab_id) + from "unnest_alias__dbt_tmp" + ); + + insert into "postgres".test_normalization."unnest_alias" ("id", "children", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_unnest_alias_hashid") + ( + select "id", "children", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_unnest_alias_hashid" + from "unnest_alias__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql new file mode 100644 index 000000000000..24d04f946c89 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."conflict_stream_array__dbt_tmp" + as ( + +-- Final base SQL model +select + "id", + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_conflict_stream_array_hashid +from "postgres"._airbyte_test_normalization."conflict_stream_array_ab3" +-- conflict_stream_array from "postgres".test_normalization._airbyte_raw_conflict_stream_array +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql new file mode 100644 index 000000000000..a664bb56d781 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."conflict_stream_name__dbt_tmp" + as ( + +-- Final base SQL model +select + "id", + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_hashid +from "postgres"._airbyte_test_normalization."conflict_stream_name_ab3" +-- conflict_stream_name from "postgres".test_normalization._airbyte_raw_conflict_stream_name +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql new file mode 100644 index 000000000000..0e06a3243fa2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."conflict_stream_name___conflict_stream_name__dbt_tmp" + as ( + +-- Final base SQL model +select + _airbyte_conflict_stream_name_2_hashid, + groups, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_3_hashid +from "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab3" +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from "postgres".test_normalization."conflict_stream_name_conflict_stream_name" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql new file mode 100644 index 000000000000..df6e327e1620 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp" + as ( + +-- Final base SQL model +select + _airbyte_conflict_stream_name_hashid, + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_2_hashid +from "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab3" +-- conflict_stream_name at conflict_stream_name/conflict_stream_name from "postgres".test_normalization."conflict_stream_name" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql new file mode 100644 index 000000000000..bdcd7a2664c1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."conflict_stream_scalar__dbt_tmp" + as ( + +-- Final base SQL model +select + "id", + conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_conflict_stream_scalar_hashid +from "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab3" +-- conflict_stream_scalar from "postgres".test_normalization._airbyte_raw_conflict_stream_scalar +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql new file mode 100644 index 000000000000..a9781fcf92d2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql @@ -0,0 +1,19 @@ + + + create table "postgres".test_normalization."nested_stream_with_c___long_names_partition__dbt_tmp" + as ( + +-- Final base SQL model +select + _airbyte_nested_stre__nto_long_names_hashid, + double_array_data, + "DATA", + "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_partition_hashid +from "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab3" +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "postgres".test_normalization."nested_stream_with_c__lting_into_long_names" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql new file mode 100644 index 000000000000..b6c629700a04 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."nested_stream_with_c___names_partition_data__dbt_tmp" + as ( + +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_data_hashid +from "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab3" +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "postgres".test_normalization."nested_stream_with_c___long_names_partition" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql new file mode 100644 index 000000000000..037c53598daf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."nested_stream_with_c__column___with__quotes__dbt_tmp" + as ( + +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_column___with__quotes_hashid +from "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab3" +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from "postgres".test_normalization."nested_stream_with_c___long_names_partition" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql new file mode 100644 index 000000000000..6d3ed538c39d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."nested_stream_with_c__ion_double_array_data__dbt_tmp" + as ( + +-- Final base SQL model +select + _airbyte_partition_hashid, + "id", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_double_array_data_hashid +from "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab3" +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "postgres".test_normalization."nested_stream_with_c___long_names_partition" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql new file mode 100644 index 000000000000..0939d42acba7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."non_nested_stream_wi__lting_into_long_names__dbt_tmp" + as ( + +-- Final base SQL model +select + "id", + "date", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_non_nested___nto_long_names_hashid +from "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab3" +-- non_nested_stream_wi__lting_into_long_names from "postgres".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql new file mode 100644 index 000000000000..5248364bc8cd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -0,0 +1,18 @@ + + + create table "postgres".test_normalization."unnest_alias_children__dbt_tmp" + as ( + +-- Final base SQL model +select + _airbyte_unnest_alias_hashid, + ab_id, + "owner", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_children_hashid +from "postgres"._airbyte_test_normalization."unnest_alias_children_ab3" +-- children at unnest_alias/children from "postgres".test_normalization."unnest_alias" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql new file mode 100644 index 000000000000..b3074c01b9c9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."unnest_alias_children_owner__dbt_tmp" + as ( + +-- Final base SQL model +select + _airbyte_children_hashid, + owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_owner_hashid +from "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab3" +-- owner at unnest_alias/children/owner from "postgres".test_normalization."unnest_alias_children" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql new file mode 100644 index 000000000000..ed8e802bc153 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization_namespace."simple_stream_with_n__lting_into_long_names__dbt_tmp" + as ( + +-- Final base SQL model +select + "id", + "date", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_simple_stre__nto_long_names_hashid +from "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab3" +-- simple_stream_with_n__lting_into_long_names from "postgres".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql new file mode 100644 index 000000000000..0d5dd2d7fcac --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql @@ -0,0 +1,20 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') || '-' || coalesce(cast("partition" as + varchar +), '') as + varchar +)) as _airbyte_nested_stre__nto_long_names_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab2" tmp +-- nested_stream_with_c__lting_into_long_names +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql deleted file mode 100644 index a0ee7b628ad3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,15 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path_text(_airbyte_data, 'name') as "name", - jsonb_extract_path_text(_airbyte_data, 'column`_''with"_quotes') as "column`_'with""_quotes", - jsonb_extract_path_text(_airbyte_data, '_ab_cdc_lsn') as _ab_cdc_lsn, - jsonb_extract_path_text(_airbyte_data, '_ab_cdc_updated_at') as _ab_cdc_updated_at, - jsonb_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at') as _ab_cdc_deleted_at, - _airbyte_emitted_at -from "postgres".test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias --- dedup_cdc_excluded - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql deleted file mode 100644 index a65dbf0f5351..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,27 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - bigint -) as "id", - cast("name" as - varchar -) as "name", - cast("column`_'with""_quotes" as - varchar -) as "column`_'with""_quotes", - cast(_ab_cdc_lsn as - float -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float -) as _ab_cdc_deleted_at, - _airbyte_emitted_at -from "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab1" --- dedup_cdc_excluded - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 96bac38f7781..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,24 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast("name" as - varchar -), '') || '-' || coalesce(cast("column`_'with""_quotes" as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_lsn as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_updated_at as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as - varchar -), '') as - varchar -)) as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab2" tmp --- dedup_cdc_excluded - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql deleted file mode 100644 index f93e05005947..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab4__dbt_tmp" as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_cdc_excluded_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab3" tmp --- dedup_cdc_excluded from "postgres".test_normalization._airbyte_raw_dedup_cdc_excluded - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql deleted file mode 100644 index 46e7afe4f771..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path_text(_airbyte_data, 'currency') as currency, - jsonb_extract_path_text(_airbyte_data, 'date') as "date", - jsonb_extract_path_text(_airbyte_data, 'timestamp_col') as timestamp_col, - jsonb_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters", - jsonb_extract_path_text(_airbyte_data, 'HKD_special___characters') as hkd_special___characters, - jsonb_extract_path_text(_airbyte_data, 'NZD') as nzd, - jsonb_extract_path_text(_airbyte_data, 'USD') as usd, - _airbyte_emitted_at -from "postgres".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias --- dedup_exchange_rate - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql deleted file mode 100644 index 52cbfdd9fae4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ /dev/null @@ -1,33 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - bigint -) as "id", - cast(currency as - varchar -) as currency, - cast(nullif("date", '') as - date -) as "date", - cast(nullif(timestamp_col, '') as - timestamp with time zone -) as timestamp_col, - cast("HKD@spéçiäl & characters" as - float -) as "HKD@spéçiäl & characters", - cast(hkd_special___characters as - varchar -) as hkd_special___characters, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - _airbyte_emitted_at -from "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab1" --- dedup_exchange_rate - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql deleted file mode 100644 index c249b790827c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql +++ /dev/null @@ -1,28 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast(currency as - varchar -), '') || '-' || coalesce(cast("date" as - varchar -), '') || '-' || coalesce(cast(timestamp_col as - varchar -), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as - varchar -), '') || '-' || coalesce(cast(hkd_special___characters as - varchar -), '') || '-' || coalesce(cast(nzd as - varchar -), '') || '-' || coalesce(cast(usd as - varchar -), '') as - varchar -)) as _airbyte_dedup_exchange_rate_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab2" tmp --- dedup_exchange_rate - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql deleted file mode 100644 index 3b69eecda1b8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab4__dbt_tmp" as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_exchange_rate_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab3" tmp --- dedup_exchange_rate from "postgres".test_normalization._airbyte_raw_dedup_exchange_rate - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab1.sql deleted file mode 100644 index a9de573688ab..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."exchange_rate_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path_text(_airbyte_data, 'currency') as currency, - jsonb_extract_path_text(_airbyte_data, 'date') as "date", - jsonb_extract_path_text(_airbyte_data, 'timestamp_col') as timestamp_col, - jsonb_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters", - jsonb_extract_path_text(_airbyte_data, 'HKD_special___characters') as hkd_special___characters, - jsonb_extract_path_text(_airbyte_data, 'NZD') as nzd, - jsonb_extract_path_text(_airbyte_data, 'USD') as usd, - _airbyte_emitted_at -from "postgres".test_normalization._airbyte_raw_exchange_rate as table_alias --- exchange_rate - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab2.sql deleted file mode 100644 index bcc55df18e53..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ /dev/null @@ -1,33 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."exchange_rate_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - bigint -) as "id", - cast(currency as - varchar -) as currency, - cast(nullif("date", '') as - date -) as "date", - cast(nullif(timestamp_col, '') as - timestamp with time zone -) as timestamp_col, - cast("HKD@spéçiäl & characters" as - float -) as "HKD@spéçiäl & characters", - cast(hkd_special___characters as - varchar -) as hkd_special___characters, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - _airbyte_emitted_at -from "postgres"._airbyte_test_normalization."exchange_rate_ab1" --- exchange_rate - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab3.sql deleted file mode 100644 index efc1e74882a1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ /dev/null @@ -1,28 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."exchange_rate_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast(currency as - varchar -), '') || '-' || coalesce(cast("date" as - varchar -), '') || '-' || coalesce(cast(timestamp_col as - varchar -), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as - varchar -), '') || '-' || coalesce(cast(hkd_special___characters as - varchar -), '') || '-' || coalesce(cast(nzd as - varchar -), '') || '-' || coalesce(cast(usd as - varchar -), '') as - varchar -)) as _airbyte_exchange_rate_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."exchange_rate_ab2" tmp --- exchange_rate - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql deleted file mode 100644 index f0ad78951449..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ /dev/null @@ -1,15 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path_text(_airbyte_data, 'name') as "name", - jsonb_extract_path_text(_airbyte_data, '_ab_cdc_lsn') as _ab_cdc_lsn, - jsonb_extract_path_text(_airbyte_data, '_ab_cdc_updated_at') as _ab_cdc_updated_at, - jsonb_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at') as _ab_cdc_deleted_at, - jsonb_extract_path_text(_airbyte_data, '_ab_cdc_log_pos') as _ab_cdc_log_pos, - _airbyte_emitted_at -from "postgres".test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias --- pos_dedup_cdcx - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql deleted file mode 100644 index 48cdc7668ca2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ /dev/null @@ -1,27 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - bigint -) as "id", - cast("name" as - varchar -) as "name", - cast(_ab_cdc_lsn as - float -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float -) as _ab_cdc_deleted_at, - cast(_ab_cdc_log_pos as - float -) as _ab_cdc_log_pos, - _airbyte_emitted_at -from "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab1" --- pos_dedup_cdcx - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index 784d9eca5762..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,24 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast("name" as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_lsn as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_updated_at as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_log_pos as - varchar -), '') as - varchar -)) as _airbyte_pos_dedup_cdcx_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab2" tmp --- pos_dedup_cdcx - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql deleted file mode 100644 index 5a9fbb127a5b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab4__dbt_tmp" as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_pos_dedup_cdcx_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab3" tmp --- pos_dedup_cdcx from "postgres".test_normalization._airbyte_raw_pos_dedup_cdcx - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index e4aea1d62935..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create table "postgres".test_normalization."dedup_cdc_excluded__dbt_tmp" - as ( - --- Final base SQL model -select - "id", - "name", - "column`_'with""_quotes", - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from "postgres".test_normalization."dedup_cdc_excluded_scd" --- dedup_cdc_excluded from "postgres".test_normalization._airbyte_raw_dedup_cdc_excluded -where _airbyte_active_row = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index c7e2d80f646c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,28 +0,0 @@ - - - create table "postgres".test_normalization."dedup_cdc_excluded_scd__dbt_tmp" - as ( - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - "id", - "name", - "column`_'with""_quotes", - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by "id" - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by "id" - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab4" --- dedup_cdc_excluded from "postgres".test_normalization._airbyte_raw_dedup_cdc_excluded -where _airbyte_row_num = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql deleted file mode 100644 index 996d92274da2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create table "postgres".test_normalization."dedup_exchange_rate__dbt_tmp" - as ( - --- Final base SQL model -select - "id", - currency, - "date", - timestamp_col, - "HKD@spéçiäl & characters", - hkd_special___characters, - nzd, - usd, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from "postgres".test_normalization."dedup_exchange_rate_scd" --- dedup_exchange_rate from "postgres".test_normalization._airbyte_raw_dedup_exchange_rate -where _airbyte_active_row = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql deleted file mode 100644 index 3ba356ddbd8a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql +++ /dev/null @@ -1,34 +0,0 @@ - - - create table "postgres".test_normalization."dedup_exchange_rate_scd__dbt_tmp" - as ( - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - "id", - currency, - "date", - timestamp_col, - "HKD@spéçiäl & characters", - hkd_special___characters, - nzd, - usd, - "date" as _airbyte_start_at, - lag("date") over ( - partition by "id", currency, cast(nzd as - varchar -) - order by "date" is null asc, "date" desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag("date") over ( - partition by "id", currency, cast(nzd as - varchar -) - order by "date" is null asc, "date" desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab4" --- dedup_exchange_rate from "postgres".test_normalization._airbyte_raw_dedup_exchange_rate -where _airbyte_row_num = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql deleted file mode 100644 index 68057cf35c7f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create table "postgres".test_normalization."pos_dedup_cdcx__dbt_tmp" - as ( - --- Final base SQL model -select - "id", - "name", - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid -from "postgres".test_normalization."pos_dedup_cdcx_scd" --- pos_dedup_cdcx from "postgres".test_normalization._airbyte_raw_pos_dedup_cdcx -where _airbyte_active_row = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index 5147b9e4d0a7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,28 +0,0 @@ - - - create table "postgres".test_normalization."pos_dedup_cdcx_scd__dbt_tmp" - as ( - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - "id", - "name", - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by "id" - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by "id" - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid -from "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab4" --- pos_dedup_cdcx from "postgres".test_normalization._airbyte_raw_pos_dedup_cdcx -where _airbyte_row_num = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..6e1c220c3cbc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -0,0 +1,18 @@ + + create view "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'name') as "name", + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_lsn') as _ab_cdc_lsn, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_updated_at') as _ab_cdc_updated_at, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at') as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias +-- dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..88c28f9fa253 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -0,0 +1,28 @@ + + create view "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + cast("name" as + varchar +) as "name", + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab1" +-- dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql new file mode 100644 index 000000000000..ed7c144f7d77 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -0,0 +1,21 @@ + + create view "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'currency') as currency, + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + jsonb_extract_path_text(_airbyte_data, 'timestamp_col') as timestamp_col, + jsonb_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters", + jsonb_extract_path_text(_airbyte_data, 'HKD_special___characters') as hkd_special___characters, + jsonb_extract_path_text(_airbyte_data, 'NZD') as nzd, + jsonb_extract_path_text(_airbyte_data, 'USD') as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql new file mode 100644 index 000000000000..7e00f6385af7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -0,0 +1,37 @@ + + create view "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + cast(currency as + varchar +) as currency, + cast(nullif("date", '') as + date +) as "date", + cast(nullif(timestamp_col, '') as + timestamp with time zone +) as timestamp_col, + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(hkd_special___characters as + varchar +) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab1" +-- dedup_exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql new file mode 100644 index 000000000000..6e6073e308e0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -0,0 +1,21 @@ + + create view "postgres"._airbyte_test_normalization."exchange_rate_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'currency') as currency, + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + jsonb_extract_path_text(_airbyte_data, 'timestamp_col') as timestamp_col, + jsonb_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters", + jsonb_extract_path_text(_airbyte_data, 'HKD_special___characters') as hkd_special___characters, + jsonb_extract_path_text(_airbyte_data, 'NZD') as nzd, + jsonb_extract_path_text(_airbyte_data, 'USD') as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql new file mode 100644 index 000000000000..6cbcedd760a2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -0,0 +1,37 @@ + + create view "postgres"._airbyte_test_normalization."exchange_rate_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + cast(currency as + varchar +) as currency, + cast(nullif("date", '') as + date +) as "date", + cast(nullif(timestamp_col, '') as + timestamp with time zone +) as timestamp_col, + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(hkd_special___characters as + varchar +) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."exchange_rate_ab1" +-- exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql new file mode 100644 index 000000000000..e2b7bfd34d06 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -0,0 +1,30 @@ + + create view "postgres"._airbyte_test_normalization."exchange_rate_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(currency as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') || '-' || coalesce(cast(timestamp_col as + varchar +), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as + varchar +), '') || '-' || coalesce(cast(hkd_special___characters as + varchar +), '') || '-' || coalesce(cast(nzd as + varchar +), '') || '-' || coalesce(cast(usd as + varchar +), '') as + varchar +)) as _airbyte_exchange_rate_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."exchange_rate_ab2" tmp +-- exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql new file mode 100644 index 000000000000..7c1fca4961ff --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -0,0 +1,18 @@ + + create view "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'name') as "name", + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_lsn') as _ab_cdc_lsn, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_updated_at') as _ab_cdc_updated_at, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at') as _ab_cdc_deleted_at, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_log_pos') as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias +-- pos_dedup_cdcx +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql new file mode 100644 index 000000000000..a9e2b29e6210 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -0,0 +1,30 @@ + + create view "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + cast("name" as + varchar +) as "name", + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + cast(_ab_cdc_log_pos as + float +) as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab1" +-- pos_dedup_cdcx +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..4380fdbedd6b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,14 @@ + + create view "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..7371209e09a7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1" +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..0f6ee557356e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,93 @@ + + + + create table "postgres".test_normalization."dedup_cdc_excluded_scd" + as ( + +with + +input_data as ( + select * + from "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab3" + -- dedup_cdc_excluded from "postgres".test_normalization._airbyte_raw_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast("id" as + varchar +), '') as + varchar +)) as _airbyte_unique_key, + "id", + "name", + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by "id" + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by "id" + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as + varchar +), cast(_ab_cdc_updated_at as + varchar +) + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(coalesce(cast(_airbyte_unique_key as + varchar +), '') || '-' || coalesce(cast(_airbyte_start_at as + varchar +), '') || '-' || coalesce(cast(_airbyte_emitted_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_updated_at as + varchar +), '') as + varchar +)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + "id", + "name", + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..68c9ff694078 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,77 @@ + + + + create table "postgres".test_normalization."renamed_dedup_cdc_excluded_scd" + as ( + +with + +input_data as ( + select * + from "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3" + -- renamed_dedup_cdc_excluded from "postgres".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast("id" as + varchar +), '') as + varchar +)) as _airbyte_unique_key, + "id", + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by "id" + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by "id" + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_renamed_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(coalesce(cast(_airbyte_unique_key as + varchar +), '') || '-' || coalesce(cast(_airbyte_start_at as + varchar +), '') || '-' || coalesce(cast(_airbyte_emitted_at as + varchar +), '') as + varchar +)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + "id", + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql new file mode 100644 index 000000000000..f19c2bcfd939 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -0,0 +1,25 @@ + + + + create table "postgres".test_normalization."dedup_cdc_excluded" + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + "id", + "name", + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from "postgres".test_normalization."dedup_cdc_excluded_scd" +-- dedup_cdc_excluded from "postgres".test_normalization._airbyte_raw_dedup_cdc_excluded +where 1 = 1 +and _airbyte_active_row = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..9fbeb8463fe0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,28 @@ + + + + create table "postgres".test_normalization."dedup_exchange_rate" + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + "id", + currency, + "date", + timestamp_col, + "HKD@spéçiäl & characters", + hkd_special___characters, + nzd, + usd, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from "postgres".test_normalization."dedup_exchange_rate_scd" +-- dedup_exchange_rate from "postgres".test_normalization._airbyte_raw_dedup_exchange_rate +where 1 = 1 +and _airbyte_active_row = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql index b0a2937dfba2..2c7e88caddef 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -16,6 +16,7 @@ select usd, _airbyte_ab_id, _airbyte_emitted_at, + now() as _airbyte_normalized_at, _airbyte_exchange_rate_hashid from "postgres"._airbyte_test_normalization."exchange_rate_ab3" -- exchange_rate from "postgres".test_normalization._airbyte_raw_exchange_rate diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..370ce4c75de2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,21 @@ + + + + create table "postgres".test_normalization."renamed_dedup_cdc_excluded" + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + "id", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from "postgres".test_normalization."renamed_dedup_cdc_excluded_scd" +-- renamed_dedup_cdc_excluded from "postgres".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded +where 1 = 1 +and _airbyte_active_row = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..48e2e8023174 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,97 @@ + + + create table "postgres".test_normalization."pos_dedup_cdcx_scd__dbt_tmp" + as ( + +with + +input_data as ( + select * + from "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab3" + -- pos_dedup_cdcx from "postgres".test_normalization._airbyte_raw_pos_dedup_cdcx +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast("id" as + varchar +), '') as + varchar +)) as _airbyte_unique_key, + "id", + "name", + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by "id" + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by "id" + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as + varchar +), cast(_ab_cdc_updated_at as + varchar +), cast(_ab_cdc_log_pos as + varchar +) + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(coalesce(cast(_airbyte_unique_key as + varchar +), '') || '-' || coalesce(cast(_airbyte_start_at as + varchar +), '') || '-' || coalesce(cast(_airbyte_emitted_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_updated_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_log_pos as + varchar +), '') as + varchar +)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + "id", + "name", + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql new file mode 100644 index 000000000000..25e9cb949467 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -0,0 +1,23 @@ + + + create table "postgres".test_normalization."pos_dedup_cdcx__dbt_tmp" + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + "id", + "name", + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from "postgres".test_normalization."pos_dedup_cdcx_scd" +-- pos_dedup_cdcx from "postgres".test_normalization._airbyte_raw_pos_dedup_cdcx +where 1 = 1 +and _airbyte_active_row = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..63d83886e5f1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,24 @@ + + create view "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast("name" as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_lsn as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_updated_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as + varchar +), '') as + varchar +)) as _airbyte_dedup_cdc_excluded_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab2" tmp +-- dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..162849517206 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,30 @@ + + create view "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(currency as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') || '-' || coalesce(cast(timestamp_col as + varchar +), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as + varchar +), '') || '-' || coalesce(cast(hkd_special___characters as + varchar +), '') || '-' || coalesce(cast(nzd as + varchar +), '') || '-' || coalesce(cast(usd as + varchar +), '') as + varchar +)) as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab2" tmp +-- dedup_exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql new file mode 100644 index 000000000000..fceb899227d1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -0,0 +1,25 @@ + + create view "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast("name" as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_lsn as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_updated_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_log_pos as + varchar +), '') as + varchar +)) as _airbyte_pos_dedup_cdcx_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab2" tmp +-- pos_dedup_cdcx +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..732f2bd1a1ea --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') as + varchar +)) as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2" tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql deleted file mode 100644 index 6144a13617e8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql +++ /dev/null @@ -1,43 +0,0 @@ -with table_row_counts as ( - select distinct count(*) as row_count, 10 as expected_count - from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} -union all - select distinct count(*) as row_count, 10 as expected_count - from {{ ref('exchange_rate') }} - -union all - - select distinct count(*) as row_count, 10 as expected_count - from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -union all - select distinct count(*) as row_count, 10 as expected_count - from {{ ref('dedup_exchange_rate_scd') }} -union all - select distinct count(*) as row_count, 5 as expected_count - from {{ ref('dedup_exchange_rate') }} - -union all - - select distinct count(*) as row_count, 8 as expected_count - from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -union all - select distinct count(*) as row_count, 8 as expected_count - from {{ ref('dedup_cdc_excluded_scd') }} -union all - select distinct count(*) as row_count, 4 as expected_count - from {{ ref('dedup_cdc_excluded') }} - -union all - - select distinct count(*) as row_count, 8 as expected_count - from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -union all - select distinct count(*) as row_count, 8 as expected_count - from {{ ref('pos_dedup_cdcx_scd') }} -union all - select distinct count(*) as row_count, 3 as expected_count - from {{ ref('pos_dedup_cdcx') }} -) -select * -from table_row_counts -where row_count != expected_count diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql index 4d6554576aca..6ff5c5605fb8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -1,13 +1,21 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as {{ adapter.quote('id') }}, {{ json_extract_scalar('_airbyte_data', ['name'], ['name']) }} as {{ adapter.quote('name') }}, - {{ json_extract_scalar('_airbyte_data', ['column`_\'with"_quotes'], ['column___with__quotes']) }} as {{ adapter.quote('column`_\'with""_quotes') }}, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_lsn'], ['_ab_cdc_lsn']) }} as _ab_cdc_lsn, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} as table_alias -- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql index e7941a7141f5..a90b1c1e09af 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -1,13 +1,21 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast({{ adapter.quote('id') }} as {{ dbt_utils.type_bigint() }}) as {{ adapter.quote('id') }}, cast({{ adapter.quote('name') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('name') }}, - cast({{ adapter.quote('column`_\'with""_quotes') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('column`_\'with""_quotes') }}, cast(_ab_cdc_lsn as {{ dbt_utils.type_float() }}) as _ab_cdc_lsn, cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('dedup_cdc_excluded_ab1') }} -- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 6a9767cf21d6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - adapter.quote('id'), - adapter.quote('name'), - adapter.quote('column`_\'with""_quotes'), - '_ab_cdc_lsn', - '_ab_cdc_updated_at', - '_ab_cdc_deleted_at', - ]) }} as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from {{ ref('dedup_cdc_excluded_ab2') }} tmp --- dedup_cdc_excluded - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql deleted file mode 100644 index aa34179c3c5c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_cdc_excluded_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('dedup_cdc_excluded_ab3') }} tmp --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql index 2703c06376ce..5575218fafd2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as {{ adapter.quote('id') }}, @@ -9,7 +14,11 @@ select {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as hkd_special___characters, {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as nzd, {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} as table_alias -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index 9b23934d2e81..74859161cd8f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast({{ adapter.quote('id') }} as {{ dbt_utils.type_bigint() }}) as {{ adapter.quote('id') }}, @@ -9,7 +14,11 @@ select cast(hkd_special___characters as {{ dbt_utils.type_string() }}) as hkd_special___characters, cast(nzd as {{ dbt_utils.type_float() }}) as nzd, cast(usd as {{ dbt_utils.type_float() }}) as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('dedup_exchange_rate_ab1') }} -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql deleted file mode 100644 index 14b695bdbb6f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - adapter.quote('id'), - 'currency', - adapter.quote('date'), - 'timestamp_col', - adapter.quote('HKD@spéçiäl & characters'), - 'hkd_special___characters', - 'nzd', - 'usd', - ]) }} as _airbyte_dedup_exchange_rate_hashid, - tmp.* -from {{ ref('dedup_exchange_rate_ab2') }} tmp --- dedup_exchange_rate - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql deleted file mode 100644 index ef5f6e52d60e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_exchange_rate_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('dedup_exchange_rate_ab3') }} tmp --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql index afce0ec584db..e0f71f0e7cf2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -1,4 +1,6 @@ {{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} @@ -13,8 +15,10 @@ select {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as nzd, {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as usd, _airbyte_ab_id, - _airbyte_emitted_at + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} as table_alias -- exchange_rate where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql index 178badcb2596..56f31c5d75f3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -1,4 +1,6 @@ {{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} @@ -13,8 +15,10 @@ select cast(nzd as {{ dbt_utils.type_float() }}) as nzd, cast(usd as {{ dbt_utils.type_float() }}) as usd, _airbyte_ab_id, - _airbyte_emitted_at + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('exchange_rate_ab1') }} -- exchange_rate where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql index 0469a220171c..8784ce3bc62d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -1,4 +1,6 @@ {{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} @@ -18,4 +20,5 @@ select from {{ ref('exchange_rate_ab2') }} tmp -- exchange_rate where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql index 436a7f04cdaa..fb9d0b0df354 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as {{ adapter.quote('id') }}, @@ -7,7 +12,10 @@ select {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_log_pos'], ['_ab_cdc_log_pos']) }} as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} as table_alias -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql index 3a27504a4c19..adf58230d903 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast({{ adapter.quote('id') }} as {{ dbt_utils.type_bigint() }}) as {{ adapter.quote('id') }}, @@ -7,7 +12,10 @@ select cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, cast(_ab_cdc_log_pos as {{ dbt_utils.type_float() }}) as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('pos_dedup_cdcx_ab1') }} -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index c3dd502cea84..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - adapter.quote('id'), - adapter.quote('name'), - '_ab_cdc_lsn', - '_ab_cdc_updated_at', - '_ab_cdc_deleted_at', - '_ab_cdc_log_pos', - ]) }} as _airbyte_pos_dedup_cdcx_hashid, - tmp.* -from {{ ref('pos_dedup_cdcx_ab2') }} tmp --- pos_dedup_cdcx - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql deleted file mode 100644 index 7c897124b395..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_pos_dedup_cdcx_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('pos_dedup_cdcx_ab3') }} tmp --- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..ca844abec811 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,17 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as {{ adapter.quote('id') }}, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..e2b60bbe8a71 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,17 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast({{ adapter.quote('id') }} as {{ dbt_utils.type_bigint() }}) as {{ adapter.quote('id') }}, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ ref('renamed_dedup_cdc_excluded_ab1') }} +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..4e4500b5ad39 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,112 @@ +{{ config( + indexes = [{'columns':['_airbyte_active_row','_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('dedup_cdc_excluded_ab3') }} + -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('dedup_cdc_excluded_ab3') }} + -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + ]) }} as _airbyte_unique_key, + {{ adapter.quote('id') }}, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by {{ adapter.quote('id') }} + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by {{ adapter.quote('id') }} + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}) + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + {{ adapter.quote('id') }}, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index ff9e861e971d..44047d79d0cd 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -1,38 +1,122 @@ {{ config( + indexes = [{'columns':['_airbyte_active_row','_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], + unique_key = "_airbyte_unique_key_scd", schema = "test_normalization", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), tags = [ "top-level" ] ) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('dedup_exchange_rate_ab3') }} + -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + 'currency', + 'nzd', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('dedup_exchange_rate_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('dedup_exchange_rate_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('dedup_exchange_rate_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('dedup_exchange_rate_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('dedup_exchange_rate_ab3') }} + -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + 'currency', + 'nzd', + ]) }} as _airbyte_unique_key, + {{ adapter.quote('id') }}, + currency, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + {{ adapter.quote('date') }} as _airbyte_start_at, + lag({{ adapter.quote('date') }}) over ( + partition by {{ adapter.quote('id') }}, currency, cast(nzd as {{ dbt_utils.type_string() }}) + order by + {{ adapter.quote('date') }} is null asc, + {{ adapter.quote('date') }} desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag({{ adapter.quote('date') }}) over ( + partition by {{ adapter.quote('id') }}, currency, cast(nzd as {{ dbt_utils.type_string() }}) + order by + {{ adapter.quote('date') }} is null asc, + {{ adapter.quote('date') }} desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_exchange_rate_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) select - {{ dbt_utils.surrogate_key([ - adapter.quote('id'), - 'currency', - 'nzd', - ]) }} as _airbyte_unique_key, - {{ adapter.quote('id') }}, - currency, - {{ adapter.quote('date') }}, - timestamp_col, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, - {{ adapter.quote('date') }} as _airbyte_start_at, - lag({{ adapter.quote('date') }}) over ( - partition by {{ adapter.quote('id') }}, currency, cast(nzd as {{ dbt_utils.type_string() }}) - order by {{ adapter.quote('date') }} is null asc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag({{ adapter.quote('date') }}) over ( - partition by {{ adapter.quote('id') }}, currency, cast(nzd as {{ dbt_utils.type_string() }}) - order by {{ adapter.quote('date') }} is null asc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from {{ ref('dedup_exchange_rate_ab4') }} --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -where 1 = 1 -and _airbyte_row_num = 1 -{{ incremental_clause('_airbyte_emitted_at') }} + _airbyte_unique_key, + _airbyte_unique_key_scd, + {{ adapter.quote('id') }}, + currency, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from dedup_data where _airbyte_row_num = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..8c57c2ee55b5 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,104 @@ +{{ config( + indexes = [{'columns':['_airbyte_active_row','_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('renamed_dedup_cdc_excluded_ab3') }} + -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('renamed_dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('renamed_dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('renamed_dedup_cdc_excluded_ab3') }} + -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + ]) }} as _airbyte_unique_key, + {{ adapter.quote('id') }}, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by {{ adapter.quote('id') }} + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by {{ adapter.quote('id') }} + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_renamed_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + {{ adapter.quote('id') }}, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql new file mode 100644 index 000000000000..18247c346368 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -0,0 +1,24 @@ +{{ config( + indexes = [{'columns':['_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + {{ adapter.quote('id') }}, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from {{ ref('dedup_cdc_excluded_scd') }} +-- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql index f31846c24b9c..370919ff628e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -1,6 +1,7 @@ {{ config( - schema = "test_normalization", + indexes = [{'columns':['_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], unique_key = "_airbyte_unique_key", + schema = "test_normalization", tags = [ "top-level" ] ) }} -- Final base SQL model @@ -16,6 +17,7 @@ select usd, _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_dedup_exchange_rate_hashid from {{ ref('dedup_exchange_rate_scd') }} -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql index 15a9bcb2ac23..b8b52c6d4d88 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql @@ -1,6 +1,7 @@ {{ config( - schema = "test_normalization", + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", tags = [ "top-level" ] ) }} -- Final base SQL model @@ -15,6 +16,7 @@ select usd, _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_exchange_rate_hashid from {{ ref('exchange_rate_ab3') }} -- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..4bf1ea58e96e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,20 @@ +{{ config( + indexes = [{'columns':['_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + {{ adapter.quote('id') }}, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from {{ ref('renamed_dedup_cdc_excluded_scd') }} +-- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..afc9f924fa86 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,114 @@ +{{ config( + indexes = [{'columns':['_airbyte_active_row','_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('pos_dedup_cdcx_ab3') }} + -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('pos_dedup_cdcx_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('pos_dedup_cdcx_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('pos_dedup_cdcx_ab3') }} + -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + ]) }} as _airbyte_unique_key, + {{ adapter.quote('id') }}, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by {{ adapter.quote('id') }} + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by {{ adapter.quote('id') }} + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_log_pos as {{ dbt_utils.type_string() }}) + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at', '_ab_cdc_log_pos' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + {{ adapter.quote('id') }}, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index 8faf5847dc66..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - {{ adapter.quote('id') }}, - {{ adapter.quote('name') }}, - {{ adapter.quote('column`_\'with""_quotes') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from {{ ref('dedup_cdc_excluded_scd') }} --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -where _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index 3924333f7df3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - {{ adapter.quote('id') }}, - {{ adapter.quote('name') }}, - {{ adapter.quote('column`_\'with""_quotes') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by {{ adapter.quote('id') }} - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by {{ adapter.quote('id') }} - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from {{ ref('dedup_cdc_excluded_ab4') }} --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql deleted file mode 100644 index a1edca0ac0b8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - {{ adapter.quote('id') }}, - currency, - {{ adapter.quote('date') }}, - timestamp_col, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from {{ ref('dedup_exchange_rate_scd') }} --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -where _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql deleted file mode 100644 index 465a1d012f31..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql +++ /dev/null @@ -1,26 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - {{ adapter.quote('id') }}, - currency, - {{ adapter.quote('date') }}, - timestamp_col, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, - {{ adapter.quote('date') }} as _airbyte_start_at, - lag({{ adapter.quote('date') }}) over ( - partition by {{ adapter.quote('id') }}, currency, cast(nzd as {{ dbt_utils.type_string() }}) - order by {{ adapter.quote('date') }} is null asc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag({{ adapter.quote('date') }}) over ( - partition by {{ adapter.quote('id') }}, currency, cast(nzd as {{ dbt_utils.type_string() }}) - order by {{ adapter.quote('date') }} is null asc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from {{ ref('dedup_exchange_rate_ab4') }} --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql index b357729508d7..04c21e5fc3be 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -1,15 +1,24 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + indexes = [{'columns':['_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select + _airbyte_unique_key, {{ adapter.quote('id') }}, {{ adapter.quote('name') }}, _ab_cdc_lsn, _ab_cdc_updated_at, _ab_cdc_deleted_at, _ab_cdc_log_pos, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_pos_dedup_cdcx_hashid from {{ ref('pos_dedup_cdcx_scd') }} -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index 6226de08b2c0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - {{ adapter.quote('id') }}, - {{ adapter.quote('name') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by {{ adapter.quote('id') }} - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by {{ adapter.quote('id') }} - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid -from {{ ref('pos_dedup_cdcx_ab4') }} --- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..e539d1844de2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,21 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + adapter.quote('name'), + '_ab_cdc_lsn', + '_ab_cdc_updated_at', + '_ab_cdc_deleted_at', + ]) }} as _airbyte_dedup_cdc_excluded_hashid, + tmp.* +from {{ ref('dedup_cdc_excluded_ab2') }} tmp +-- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..ee7631c2c6ea --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,24 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + 'currency', + adapter.quote('date'), + 'timestamp_col', + adapter.quote('HKD@spéçiäl & characters'), + 'hkd_special___characters', + 'nzd', + 'usd', + ]) }} as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from {{ ref('dedup_exchange_rate_ab2') }} tmp +-- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql new file mode 100644 index 000000000000..9da3c6fea93a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -0,0 +1,21 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + adapter.quote('name'), + '_ab_cdc_lsn', + '_ab_cdc_updated_at', + '_ab_cdc_deleted_at', + '_ab_cdc_log_pos', + ]) }} as _airbyte_pos_dedup_cdcx_hashid, + tmp.* +from {{ ref('pos_dedup_cdcx_ab2') }} tmp +-- pos_dedup_cdcx +where 1 = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..b152709f2db5 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,17 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + ]) }} as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from {{ ref('renamed_dedup_cdc_excluded_ab2') }} tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/sources.yml index c085ca8ccc25..45c338b893ca 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/sources.yml @@ -10,3 +10,4 @@ sources: - name: _airbyte_raw_dedup_exchange_rate - name: _airbyte_raw_exchange_rate - name: _airbyte_raw_pos_dedup_cdcx + - name: _airbyte_raw_renamed_dedup_cdc_excluded diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..6e1c220c3cbc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -0,0 +1,18 @@ + + create view "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'name') as "name", + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_lsn') as _ab_cdc_lsn, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_updated_at') as _ab_cdc_updated_at, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at') as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias +-- dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..88c28f9fa253 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -0,0 +1,28 @@ + + create view "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + cast("name" as + varchar +) as "name", + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab1" +-- dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql new file mode 100644 index 000000000000..ed7c144f7d77 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -0,0 +1,21 @@ + + create view "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'currency') as currency, + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + jsonb_extract_path_text(_airbyte_data, 'timestamp_col') as timestamp_col, + jsonb_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters", + jsonb_extract_path_text(_airbyte_data, 'HKD_special___characters') as hkd_special___characters, + jsonb_extract_path_text(_airbyte_data, 'NZD') as nzd, + jsonb_extract_path_text(_airbyte_data, 'USD') as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql new file mode 100644 index 000000000000..7e00f6385af7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -0,0 +1,37 @@ + + create view "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + cast(currency as + varchar +) as currency, + cast(nullif("date", '') as + date +) as "date", + cast(nullif(timestamp_col, '') as + timestamp with time zone +) as timestamp_col, + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(hkd_special___characters as + varchar +) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab1" +-- dedup_exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql new file mode 100644 index 000000000000..6e6073e308e0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -0,0 +1,21 @@ + + create view "postgres"._airbyte_test_normalization."exchange_rate_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'currency') as currency, + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + jsonb_extract_path_text(_airbyte_data, 'timestamp_col') as timestamp_col, + jsonb_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters", + jsonb_extract_path_text(_airbyte_data, 'HKD_special___characters') as hkd_special___characters, + jsonb_extract_path_text(_airbyte_data, 'NZD') as nzd, + jsonb_extract_path_text(_airbyte_data, 'USD') as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql new file mode 100644 index 000000000000..6cbcedd760a2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -0,0 +1,37 @@ + + create view "postgres"._airbyte_test_normalization."exchange_rate_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + cast(currency as + varchar +) as currency, + cast(nullif("date", '') as + date +) as "date", + cast(nullif(timestamp_col, '') as + timestamp with time zone +) as timestamp_col, + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(hkd_special___characters as + varchar +) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."exchange_rate_ab1" +-- exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql new file mode 100644 index 000000000000..e2b7bfd34d06 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -0,0 +1,30 @@ + + create view "postgres"._airbyte_test_normalization."exchange_rate_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(currency as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') || '-' || coalesce(cast(timestamp_col as + varchar +), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as + varchar +), '') || '-' || coalesce(cast(hkd_special___characters as + varchar +), '') || '-' || coalesce(cast(nzd as + varchar +), '') || '-' || coalesce(cast(usd as + varchar +), '') as + varchar +)) as _airbyte_exchange_rate_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."exchange_rate_ab2" tmp +-- exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql new file mode 100644 index 000000000000..7c1fca4961ff --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -0,0 +1,18 @@ + + create view "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'name') as "name", + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_lsn') as _ab_cdc_lsn, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_updated_at') as _ab_cdc_updated_at, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at') as _ab_cdc_deleted_at, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_log_pos') as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias +-- pos_dedup_cdcx +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql new file mode 100644 index 000000000000..a9e2b29e6210 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -0,0 +1,30 @@ + + create view "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + cast("name" as + varchar +) as "name", + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + cast(_ab_cdc_log_pos as + float +) as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab1" +-- pos_dedup_cdcx +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..4380fdbedd6b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,14 @@ + + create view "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..7371209e09a7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1" +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..ff39afab32ca --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,14 @@ + + delete + from "postgres".test_normalization."dedup_cdc_excluded_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "dedup_cdc_excluded_scd__dbt_tmp" + ); + + insert into "postgres".test_normalization."dedup_cdc_excluded_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid" + from "dedup_cdc_excluded_scd__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..5536e95b3075 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,14 @@ + + delete + from "postgres".test_normalization."renamed_dedup_cdc_excluded_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "renamed_dedup_cdc_excluded_scd__dbt_tmp" + ); + + insert into "postgres".test_normalization."renamed_dedup_cdc_excluded_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid" + from "renamed_dedup_cdc_excluded_scd__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql new file mode 100644 index 000000000000..c905b883e251 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -0,0 +1,14 @@ + + delete + from "postgres".test_normalization."dedup_cdc_excluded" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "dedup_cdc_excluded__dbt_tmp" + ); + + insert into "postgres".test_normalization."dedup_cdc_excluded" ("_airbyte_unique_key", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid") + ( + select "_airbyte_unique_key", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid" + from "dedup_cdc_excluded__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..cc0c70d79775 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,14 @@ + + delete + from "postgres".test_normalization."dedup_exchange_rate" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "dedup_exchange_rate__dbt_tmp" + ); + + insert into "postgres".test_normalization."dedup_exchange_rate" ("_airbyte_unique_key", "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") + ( + select "_airbyte_unique_key", "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid" + from "dedup_exchange_rate__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql index 49f01c196e0f..4e20aa65740e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -6,9 +6,9 @@ from "exchange_rate__dbt_tmp" ); - insert into "postgres".test_normalization."exchange_rate" ("id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_exchange_rate_hashid") + insert into "postgres".test_normalization."exchange_rate" ("id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid") ( - select "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_exchange_rate_hashid" + select "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid" from "exchange_rate__dbt_tmp" ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..a5cc40567b2c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,14 @@ + + delete + from "postgres".test_normalization."renamed_dedup_cdc_excluded" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "renamed_dedup_cdc_excluded__dbt_tmp" + ); + + insert into "postgres".test_normalization."renamed_dedup_cdc_excluded" ("_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid") + ( + select "_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid" + from "renamed_dedup_cdc_excluded__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..48e2e8023174 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,97 @@ + + + create table "postgres".test_normalization."pos_dedup_cdcx_scd__dbt_tmp" + as ( + +with + +input_data as ( + select * + from "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab3" + -- pos_dedup_cdcx from "postgres".test_normalization._airbyte_raw_pos_dedup_cdcx +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast("id" as + varchar +), '') as + varchar +)) as _airbyte_unique_key, + "id", + "name", + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by "id" + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by "id" + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as + varchar +), cast(_ab_cdc_updated_at as + varchar +), cast(_ab_cdc_log_pos as + varchar +) + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(coalesce(cast(_airbyte_unique_key as + varchar +), '') || '-' || coalesce(cast(_airbyte_start_at as + varchar +), '') || '-' || coalesce(cast(_airbyte_emitted_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_updated_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_log_pos as + varchar +), '') as + varchar +)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + "id", + "name", + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql new file mode 100644 index 000000000000..25e9cb949467 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -0,0 +1,23 @@ + + + create table "postgres".test_normalization."pos_dedup_cdcx__dbt_tmp" + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + "id", + "name", + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from "postgres".test_normalization."pos_dedup_cdcx_scd" +-- pos_dedup_cdcx from "postgres".test_normalization._airbyte_raw_pos_dedup_cdcx +where 1 = 1 +and _airbyte_active_row = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..63d83886e5f1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,24 @@ + + create view "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast("name" as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_lsn as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_updated_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as + varchar +), '') as + varchar +)) as _airbyte_dedup_cdc_excluded_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab2" tmp +-- dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..162849517206 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,30 @@ + + create view "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(currency as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') || '-' || coalesce(cast(timestamp_col as + varchar +), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as + varchar +), '') || '-' || coalesce(cast(hkd_special___characters as + varchar +), '') || '-' || coalesce(cast(nzd as + varchar +), '') || '-' || coalesce(cast(usd as + varchar +), '') as + varchar +)) as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab2" tmp +-- dedup_exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql new file mode 100644 index 000000000000..fceb899227d1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -0,0 +1,25 @@ + + create view "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast("name" as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_lsn as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_updated_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_log_pos as + varchar +), '') as + varchar +)) as _airbyte_pos_dedup_cdcx_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab2" tmp +-- pos_dedup_cdcx +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..732f2bd1a1ea --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') as + varchar +)) as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2" tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql new file mode 100644 index 000000000000..5fc6f0070232 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -0,0 +1,21 @@ + + create view "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'currency') as currency, + jsonb_extract_path_text(_airbyte_data, 'new_column') as new_column, + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + jsonb_extract_path_text(_airbyte_data, 'timestamp_col') as timestamp_col, + jsonb_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters", + jsonb_extract_path_text(_airbyte_data, 'NZD') as nzd, + jsonb_extract_path_text(_airbyte_data, 'USD') as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql new file mode 100644 index 000000000000..1f848d1a326a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -0,0 +1,37 @@ + + create view "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + float +) as "id", + cast(currency as + varchar +) as currency, + cast(new_column as + float +) as new_column, + cast(nullif("date", '') as + date +) as "date", + cast(nullif(timestamp_col, '') as + timestamp with time zone +) as timestamp_col, + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(nzd as + float +) as nzd, + cast(usd as + bigint +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab1" +-- dedup_exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql new file mode 100644 index 000000000000..53eb0e732149 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -0,0 +1,21 @@ + + create view "postgres"._airbyte_test_normalization."exchange_rate_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'currency') as currency, + jsonb_extract_path_text(_airbyte_data, 'new_column') as new_column, + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + jsonb_extract_path_text(_airbyte_data, 'timestamp_col') as timestamp_col, + jsonb_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters", + jsonb_extract_path_text(_airbyte_data, 'NZD') as nzd, + jsonb_extract_path_text(_airbyte_data, 'USD') as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql new file mode 100644 index 000000000000..6b8c0fa4997e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -0,0 +1,37 @@ + + create view "postgres"._airbyte_test_normalization."exchange_rate_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + float +) as "id", + cast(currency as + varchar +) as currency, + cast(new_column as + float +) as new_column, + cast(nullif("date", '') as + date +) as "date", + cast(nullif(timestamp_col, '') as + timestamp with time zone +) as timestamp_col, + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."exchange_rate_ab1" +-- exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql new file mode 100644 index 000000000000..da83934e7c93 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -0,0 +1,30 @@ + + create view "postgres"._airbyte_test_normalization."exchange_rate_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(currency as + varchar +), '') || '-' || coalesce(cast(new_column as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') || '-' || coalesce(cast(timestamp_col as + varchar +), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as + varchar +), '') || '-' || coalesce(cast(nzd as + varchar +), '') || '-' || coalesce(cast(usd as + varchar +), '') as + varchar +)) as _airbyte_exchange_rate_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."exchange_rate_ab2" tmp +-- exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..b3923b59c83c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,18 @@ + + create view "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'name') as "name", + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_lsn') as _ab_cdc_lsn, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_updated_at') as _ab_cdc_updated_at, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at') as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..aa3cc798fcf9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,28 @@ + + create view "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + cast("name" as + varchar +) as "name", + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1" +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..7693af7ef2e6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,14 @@ + + delete + from "postgres".test_normalization."renamed_dedup_cdc_excluded_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "renamed_dedup_cdc_excluded_scd__dbt_tmp" + ); + + insert into "postgres".test_normalization."renamed_dedup_cdc_excluded_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at" + from "renamed_dedup_cdc_excluded_scd__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..ecc81c8883b0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,14 @@ + + delete + from "postgres".test_normalization."dedup_exchange_rate" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "dedup_exchange_rate__dbt_tmp" + ); + + insert into "postgres".test_normalization."dedup_exchange_rate" ("_airbyte_unique_key", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid", "new_column", "id") + ( + select "_airbyte_unique_key", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid", "new_column", "id" + from "dedup_exchange_rate__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..381151b8af8c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -0,0 +1,14 @@ + + delete + from "postgres".test_normalization."exchange_rate" + where (_airbyte_ab_id) in ( + select (_airbyte_ab_id) + from "exchange_rate__dbt_tmp" + ); + + insert into "postgres".test_normalization."exchange_rate" ("currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid", "new_column", "id") + ( + select "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid", "new_column", "id" + from "exchange_rate__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..c8edd1056dd5 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,14 @@ + + delete + from "postgres".test_normalization."renamed_dedup_cdc_excluded" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "renamed_dedup_cdc_excluded__dbt_tmp" + ); + + insert into "postgres".test_normalization."renamed_dedup_cdc_excluded" ("_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at") + ( + select "_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at" + from "renamed_dedup_cdc_excluded__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..21806fa208d3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,30 @@ + + create view "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(currency as + varchar +), '') || '-' || coalesce(cast(new_column as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') || '-' || coalesce(cast(timestamp_col as + varchar +), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as + varchar +), '') || '-' || coalesce(cast(nzd as + varchar +), '') || '-' || coalesce(cast(usd as + varchar +), '') as + varchar +)) as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab2" tmp +-- dedup_exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..b7b5f10b73a7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,24 @@ + + create view "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast("name" as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_lsn as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_updated_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as + varchar +), '') as + varchar +)) as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2" tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql deleted file mode 100644 index fba34de13a3c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql +++ /dev/null @@ -1,43 +0,0 @@ - - - create table - "integrationtests".test_normalization."conflict_stream_array__dbt_tmp" - - - as ( - -with __dbt__cte__conflict_stream_array_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - json_extract_path_text(_airbyte_data, 'conflict_stream_array', true) as conflict_stream_array, - _airbyte_emitted_at -from "integrationtests".test_normalization._airbyte_raw_conflict_stream_array as table_alias --- conflict_stream_array -), __dbt__cte__conflict_stream_array_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as varchar) as id, - conflict_stream_array, - _airbyte_emitted_at -from __dbt__cte__conflict_stream_array_ab1 --- conflict_stream_array -), __dbt__cte__conflict_stream_array_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(conflict_stream_array as varchar), '') as varchar)) as _airbyte_conflict_stream_array_hashid, - tmp.* -from __dbt__cte__conflict_stream_array_ab2 tmp --- conflict_stream_array -)-- Final base SQL model -select - id, - conflict_stream_array, - _airbyte_emitted_at, - _airbyte_conflict_stream_array_hashid -from __dbt__cte__conflict_stream_array_ab3 --- conflict_stream_array from "integrationtests".test_normalization._airbyte_raw_conflict_stream_array - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql deleted file mode 100644 index d23691bcb14a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql +++ /dev/null @@ -1,45 +0,0 @@ - - - create table - "integrationtests".test_normalization."conflict_stream_name__dbt_tmp" - - - as ( - -with __dbt__cte__conflict_stream_name_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - - case when json_extract_path_text(table_alias._airbyte_data, 'conflict_stream_name', true) != '' then json_extract_path_text(table_alias._airbyte_data, 'conflict_stream_name', true) end - as conflict_stream_name, - _airbyte_emitted_at -from "integrationtests".test_normalization._airbyte_raw_conflict_stream_name as table_alias --- conflict_stream_name -), __dbt__cte__conflict_stream_name_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as varchar) as id, - cast(conflict_stream_name as varchar) as conflict_stream_name, - _airbyte_emitted_at -from __dbt__cte__conflict_stream_name_ab1 --- conflict_stream_name -), __dbt__cte__conflict_stream_name_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(conflict_stream_name as varchar), '') as varchar)) as _airbyte_conflict_stream_name_hashid, - tmp.* -from __dbt__cte__conflict_stream_name_ab2 tmp --- conflict_stream_name -)-- Final base SQL model -select - id, - conflict_stream_name, - _airbyte_emitted_at, - _airbyte_conflict_stream_name_hashid -from __dbt__cte__conflict_stream_name_ab3 --- conflict_stream_name from "integrationtests".test_normalization._airbyte_raw_conflict_stream_name - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql deleted file mode 100644 index e6ee9e646b1f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ /dev/null @@ -1,46 +0,0 @@ - - - create table - "integrationtests".test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp" - - - as ( - -with __dbt__cte__conflict_stream_name_conflict_stream_name_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_hashid, - - case when json_extract_path_text(table_alias.conflict_stream_name, 'conflict_stream_name', true) != '' then json_extract_path_text(table_alias.conflict_stream_name, 'conflict_stream_name', true) end - as conflict_stream_name, - _airbyte_emitted_at -from "integrationtests".test_normalization."conflict_stream_name" as table_alias -where conflict_stream_name is not null --- conflict_stream_name at conflict_stream_name/conflict_stream_name -), __dbt__cte__conflict_stream_name_conflict_stream_name_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_hashid, - cast(conflict_stream_name as varchar) as conflict_stream_name, - _airbyte_emitted_at -from __dbt__cte__conflict_stream_name_conflict_stream_name_ab1 --- conflict_stream_name at conflict_stream_name/conflict_stream_name -), __dbt__cte__conflict_stream_name_conflict_stream_name_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_conflict_stream_name_hashid as varchar), '') || '-' || coalesce(cast(conflict_stream_name as varchar), '') as varchar)) as _airbyte_conflict_stream_name_2_hashid, - tmp.* -from __dbt__cte__conflict_stream_name_conflict_stream_name_ab2 tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name -)-- Final base SQL model -select - _airbyte_conflict_stream_name_hashid, - conflict_stream_name, - _airbyte_emitted_at, - _airbyte_conflict_stream_name_2_hashid -from __dbt__cte__conflict_stream_name_conflict_stream_name_ab3 --- conflict_stream_name at conflict_stream_name/conflict_stream_name from "integrationtests".test_normalization."conflict_stream_name" - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql deleted file mode 100644 index d49f154857f6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql +++ /dev/null @@ -1,44 +0,0 @@ - - - create table - "integrationtests".test_normalization."conflict_stream_name_conflict_stream_name_conflict_stream_name__dbt_tmp" - - - as ( - -with __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_2_hashid, - case when json_extract_path_text(conflict_stream_name, 'groups', true) != '' then json_extract_path_text(conflict_stream_name, 'groups', true) end as groups, - _airbyte_emitted_at -from "integrationtests".test_normalization."conflict_stream_name_conflict_stream_name" as table_alias -where conflict_stream_name is not null --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -), __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_2_hashid, - cast(groups as varchar) as groups, - _airbyte_emitted_at -from __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1 --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -), __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as varchar), '') || '-' || coalesce(cast(groups as varchar), '') as varchar)) as _airbyte_conflict_stream_name_3_hashid, - tmp.* -from __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2 tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -)-- Final base SQL model -select - _airbyte_conflict_stream_name_2_hashid, - groups, - _airbyte_emitted_at, - _airbyte_conflict_stream_name_3_hashid -from __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3 --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from "integrationtests".test_normalization."conflict_stream_name_conflict_stream_name" - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql deleted file mode 100644 index 9c4da3e16814..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ /dev/null @@ -1,45 +0,0 @@ - - - create table - "integrationtests".test_normalization."conflict_stream_scalar__dbt_tmp" - - - as ( - -with __dbt__cte__conflict_stream_scalar_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - case when json_extract_path_text(_airbyte_data, 'conflict_stream_scalar', true) != '' then json_extract_path_text(_airbyte_data, 'conflict_stream_scalar', true) end as conflict_stream_scalar, - _airbyte_emitted_at -from "integrationtests".test_normalization._airbyte_raw_conflict_stream_scalar as table_alias --- conflict_stream_scalar -), __dbt__cte__conflict_stream_scalar_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as varchar) as id, - cast(conflict_stream_scalar as - bigint -) as conflict_stream_scalar, - _airbyte_emitted_at -from __dbt__cte__conflict_stream_scalar_ab1 --- conflict_stream_scalar -), __dbt__cte__conflict_stream_scalar_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(conflict_stream_scalar as varchar), '') as varchar)) as _airbyte_conflict_stream_scalar_hashid, - tmp.* -from __dbt__cte__conflict_stream_scalar_ab2 tmp --- conflict_stream_scalar -)-- Final base SQL model -select - id, - conflict_stream_scalar, - _airbyte_emitted_at, - _airbyte_conflict_stream_scalar_hashid -from __dbt__cte__conflict_stream_scalar_ab3 --- conflict_stream_scalar from "integrationtests".test_normalization._airbyte_raw_conflict_stream_scalar - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql deleted file mode 100644 index f83a6b593845..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create table - "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names__dbt_tmp" - - - as ( - --- Final base SQL model -select - id, - date, - "partition", - _airbyte_emitted_at, - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid -from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_scd" --- nested_stream_with_complex_columns_resulting_into_long_names from "integrationtests".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names -where _airbyte_active_row = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql deleted file mode 100644 index 13de69326ca0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql +++ /dev/null @@ -1,50 +0,0 @@ - - - create table - "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition__dbt_tmp" - - - as ( - -with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, - json_extract_path_text("partition", 'double_array_data', true) as double_array_data, - json_extract_path_text("partition", 'DATA', true) as data, - json_extract_path_text("partition", 'column`_''with"_quotes', true) as "column`_'with""_quotes", - _airbyte_emitted_at -from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names" as table_alias -where "partition" is not null --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, - double_array_data, - data, - "column`_'with""_quotes", - _airbyte_emitted_at -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1 --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid as varchar), '') || '-' || coalesce(cast(double_array_data as varchar), '') || '-' || coalesce(cast(data as varchar), '') || '-' || coalesce(cast("column`_'with""_quotes" as varchar), '') as varchar)) as _airbyte_partition_hashid, - tmp.* -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2 tmp --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -)-- Final base SQL model -select - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, - double_array_data, - data, - "column`_'with""_quotes", - _airbyte_emitted_at, - _airbyte_partition_hashid -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3 --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names" - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql deleted file mode 100644 index 2c218b388b5d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql +++ /dev/null @@ -1,89 +0,0 @@ - - - create table - "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes__dbt_tmp" - - - as ( - -with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -with numbers as ( - - - - - with p as ( - select 0 as generated_number union all select 1 - ), unioned as ( - - select - - - p0.generated_number * power(2, 0) - - - + 1 - as generated_number - - from - - - p as p0 - - - - ) - - select * - from unioned - where generated_number <= 1 - order by generated_number - - -), -joined as ( - select - _airbyte_partition_hashid as _airbyte_hashid, - json_extract_array_element_text("column`_'with""_quotes", numbers.generated_number::int - 1, true) as _airbyte_nested_data - from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" - cross join numbers - -- only generate the number of records in the cross join that corresponds - -- to the number of items in nested_stream_with_complex_columns_resulting_into_long_names_partition."column`_'with""_quotes" - where numbers.generated_number <= json_array_length("column`_'with""_quotes", true) -) -select - _airbyte_partition_hashid, - case when json_extract_path_text(_airbyte_nested_data, 'currency', true) != '' then json_extract_path_text(_airbyte_nested_data, 'currency', true) end as currency, - _airbyte_emitted_at -from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias -left join joined on _airbyte_partition_hashid = joined._airbyte_hashid -where "column`_'with""_quotes" is not null --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as varchar) as currency, - _airbyte_emitted_at -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1 --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_partition_hashid as varchar), '') || '-' || coalesce(cast(currency as varchar), '') as varchar)) as _airbyte_column___with__quotes_hashid, - tmp.* -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2 tmp --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -)-- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_emitted_at, - _airbyte_column___with__quotes_hashid -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3 --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql deleted file mode 100644 index 50b9df98fc43..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql +++ /dev/null @@ -1,89 +0,0 @@ - - - create table - "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition_data__dbt_tmp" - - - as ( - -with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -with numbers as ( - - - - - with p as ( - select 0 as generated_number union all select 1 - ), unioned as ( - - select - - - p0.generated_number * power(2, 0) - - - + 1 - as generated_number - - from - - - p as p0 - - - - ) - - select * - from unioned - where generated_number <= 1 - order by generated_number - - -), -joined as ( - select - _airbyte_partition_hashid as _airbyte_hashid, - json_extract_array_element_text(data, numbers.generated_number::int - 1, true) as _airbyte_nested_data - from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" - cross join numbers - -- only generate the number of records in the cross join that corresponds - -- to the number of items in nested_stream_with_complex_columns_resulting_into_long_names_partition.data - where numbers.generated_number <= json_array_length(data, true) -) -select - _airbyte_partition_hashid, - case when json_extract_path_text(_airbyte_nested_data, 'currency', true) != '' then json_extract_path_text(_airbyte_nested_data, 'currency', true) end as currency, - _airbyte_emitted_at -from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias -left join joined on _airbyte_partition_hashid = joined._airbyte_hashid -where data is not null --- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as varchar) as currency, - _airbyte_emitted_at -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1 --- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_partition_hashid as varchar), '') || '-' || coalesce(cast(currency as varchar), '') as varchar)) as _airbyte_data_hashid, - tmp.* -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2 tmp --- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -)-- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_emitted_at, - _airbyte_data_hashid -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3 --- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql deleted file mode 100644 index 6c3f2811c690..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql +++ /dev/null @@ -1,89 +0,0 @@ - - - create table - "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data__dbt_tmp" - - - as ( - -with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -with numbers as ( - - - - - with p as ( - select 0 as generated_number union all select 1 - ), unioned as ( - - select - - - p0.generated_number * power(2, 0) - - - + 1 - as generated_number - - from - - - p as p0 - - - - ) - - select * - from unioned - where generated_number <= 2 - order by generated_number - - -), -joined as ( - select - _airbyte_partition_hashid as _airbyte_hashid, - json_extract_array_element_text(double_array_data, numbers.generated_number::int - 1, true) as _airbyte_nested_data - from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" - cross join numbers - -- only generate the number of records in the cross join that corresponds - -- to the number of items in nested_stream_with_complex_columns_resulting_into_long_names_partition.double_array_data - where numbers.generated_number <= json_array_length(double_array_data, true) -) -select - _airbyte_partition_hashid, - case when json_extract_path_text(_airbyte_nested_data, 'id', true) != '' then json_extract_path_text(_airbyte_nested_data, 'id', true) end as id, - _airbyte_emitted_at -from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias -left join joined on _airbyte_partition_hashid = joined._airbyte_hashid -where double_array_data is not null --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(id as varchar) as id, - _airbyte_emitted_at -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1 --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_partition_hashid as varchar), '') || '-' || coalesce(cast(id as varchar), '') as varchar)) as _airbyte_double_array_data_hashid, - tmp.* -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2 tmp --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -)-- Final base SQL model -select - _airbyte_partition_hashid, - id, - _airbyte_emitted_at, - _airbyte_double_array_data_hashid -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3 --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql deleted file mode 100644 index 2ff25efaa5dc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql +++ /dev/null @@ -1,69 +0,0 @@ - - - create table - "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_scd__dbt_tmp" - - - as ( - -with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, - - case when json_extract_path_text(table_alias._airbyte_data, 'partition', true) != '' then json_extract_path_text(table_alias._airbyte_data, 'partition', true) end - as "partition", - _airbyte_emitted_at -from "integrationtests".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias --- nested_stream_with_complex_columns_resulting_into_long_names -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as varchar) as id, - cast(date as varchar) as date, - cast("partition" as varchar) as "partition", - _airbyte_emitted_at -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab1 --- nested_stream_with_complex_columns_resulting_into_long_names -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(date as varchar), '') || '-' || coalesce(cast("partition" as varchar), '') as varchar)) as _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, - tmp.* -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab2 tmp --- nested_stream_with_complex_columns_resulting_into_long_names -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab4 as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab3 tmp --- nested_stream_with_complex_columns_resulting_into_long_names from "integrationtests".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names -)-- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - date, - "partition", - date as _airbyte_start_at, - lag(date) over ( - partition by id - order by date is null asc, date desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(date) over ( - partition by id - order by date is null asc, date desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab4 --- nested_stream_with_complex_columns_resulting_into_long_names from "integrationtests".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names -where _airbyte_row_num = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql deleted file mode 100644 index 94087ad12a9c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql +++ /dev/null @@ -1,43 +0,0 @@ - - - create table - "integrationtests".test_normalization."non_nested_stream_without_namespace_resulting_into_long_names__dbt_tmp" - - - as ( - -with __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, - _airbyte_emitted_at -from "integrationtests".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias --- non_nested_stream_without_namespace_resulting_into_long_names -), __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as varchar) as id, - cast(date as varchar) as date, - _airbyte_emitted_at -from __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab1 --- non_nested_stream_without_namespace_resulting_into_long_names -), __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(date as varchar), '') as varchar)) as _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid, - tmp.* -from __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab2 tmp --- non_nested_stream_without_namespace_resulting_into_long_names -)-- Final base SQL model -select - id, - date, - _airbyte_emitted_at, - _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid -from __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab3 --- non_nested_stream_without_namespace_resulting_into_long_names from "integrationtests".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql deleted file mode 100644 index e11bd8416763..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,45 +0,0 @@ - - - create table - "integrationtests".test_normalization."unnest_alias__dbt_tmp" - - - as ( - -with __dbt__cte__unnest_alias_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - json_extract_path_text(_airbyte_data, 'children', true) as children, - _airbyte_emitted_at -from "integrationtests".test_normalization._airbyte_raw_unnest_alias as table_alias --- unnest_alias -), __dbt__cte__unnest_alias_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - children, - _airbyte_emitted_at -from __dbt__cte__unnest_alias_ab1 --- unnest_alias -), __dbt__cte__unnest_alias_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(children as varchar), '') as varchar)) as _airbyte_unnest_alias_hashid, - tmp.* -from __dbt__cte__unnest_alias_ab2 tmp --- unnest_alias -)-- Final base SQL model -select - id, - children, - _airbyte_emitted_at, - _airbyte_unnest_alias_hashid -from __dbt__cte__unnest_alias_ab3 --- unnest_alias from "integrationtests".test_normalization._airbyte_raw_unnest_alias - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql deleted file mode 100644 index 8bc518e99add..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql +++ /dev/null @@ -1,96 +0,0 @@ - - - create table - "integrationtests".test_normalization."unnest_alias_children__dbt_tmp" - - - as ( - -with __dbt__cte__unnest_alias_children_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -with numbers as ( - - - - - with p as ( - select 0 as generated_number union all select 1 - ), unioned as ( - - select - - - p0.generated_number * power(2, 0) - - - + 1 - as generated_number - - from - - - p as p0 - - - - ) - - select * - from unioned - where generated_number <= 2 - order by generated_number - - -), -joined as ( - select - _airbyte_unnest_alias_hashid as _airbyte_hashid, - json_extract_array_element_text(children, numbers.generated_number::int - 1, true) as _airbyte_nested_data - from "integrationtests".test_normalization."unnest_alias" - cross join numbers - -- only generate the number of records in the cross join that corresponds - -- to the number of items in unnest_alias.children - where numbers.generated_number <= json_array_length(children, true) -) -select - _airbyte_unnest_alias_hashid, - case when json_extract_path_text(_airbyte_nested_data, 'ab_id', true) != '' then json_extract_path_text(_airbyte_nested_data, 'ab_id', true) end as ab_id, - - case when json_extract_path_text(_airbyte_nested_data, 'owner', true) != '' then json_extract_path_text(_airbyte_nested_data, 'owner', true) end - as owner, - _airbyte_emitted_at -from "integrationtests".test_normalization."unnest_alias" as table_alias -left join joined on _airbyte_unnest_alias_hashid = joined._airbyte_hashid -where children is not null --- children at unnest_alias/children -), __dbt__cte__unnest_alias_children_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_unnest_alias_hashid, - cast(ab_id as - bigint -) as ab_id, - cast(owner as varchar) as owner, - _airbyte_emitted_at -from __dbt__cte__unnest_alias_children_ab1 --- children at unnest_alias/children -), __dbt__cte__unnest_alias_children_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_unnest_alias_hashid as varchar), '') || '-' || coalesce(cast(ab_id as varchar), '') || '-' || coalesce(cast(owner as varchar), '') as varchar)) as _airbyte_children_hashid, - tmp.* -from __dbt__cte__unnest_alias_children_ab2 tmp --- children at unnest_alias/children -)-- Final base SQL model -select - _airbyte_unnest_alias_hashid, - ab_id, - owner, - _airbyte_emitted_at, - _airbyte_children_hashid -from __dbt__cte__unnest_alias_children_ab3 --- children at unnest_alias/children from "integrationtests".test_normalization."unnest_alias" - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql deleted file mode 100644 index b385cbabb620..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ /dev/null @@ -1,46 +0,0 @@ - - - create table - "integrationtests".test_normalization."unnest_alias_children_owner__dbt_tmp" - - - as ( - -with __dbt__cte__unnest_alias_children_owner_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_children_hashid, - case when json_extract_path_text(owner, 'owner_id', true) != '' then json_extract_path_text(owner, 'owner_id', true) end as owner_id, - _airbyte_emitted_at -from "integrationtests".test_normalization."unnest_alias_children" as table_alias -where owner is not null --- owner at unnest_alias/children/owner -), __dbt__cte__unnest_alias_children_owner_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_children_hashid, - cast(owner_id as - bigint -) as owner_id, - _airbyte_emitted_at -from __dbt__cte__unnest_alias_children_owner_ab1 --- owner at unnest_alias/children/owner -), __dbt__cte__unnest_alias_children_owner_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_children_hashid as varchar), '') || '-' || coalesce(cast(owner_id as varchar), '') as varchar)) as _airbyte_owner_hashid, - tmp.* -from __dbt__cte__unnest_alias_children_owner_ab2 tmp --- owner at unnest_alias/children/owner -)-- Final base SQL model -select - _airbyte_children_hashid, - owner_id, - _airbyte_emitted_at, - _airbyte_owner_hashid -from __dbt__cte__unnest_alias_children_owner_ab3 --- owner at unnest_alias/children/owner from "integrationtests".test_normalization."unnest_alias_children" - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql deleted file mode 100644 index 6627c7dcc0a1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql +++ /dev/null @@ -1,43 +0,0 @@ - - - create table - "integrationtests".test_normalization_namespace."simple_stream_with_namespace_resulting_into_long_names__dbt_tmp" - - - as ( - -with __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, - _airbyte_emitted_at -from "integrationtests".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias --- simple_stream_with_namespace_resulting_into_long_names -), __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as varchar) as id, - cast(date as varchar) as date, - _airbyte_emitted_at -from __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab1 --- simple_stream_with_namespace_resulting_into_long_names -), __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(date as varchar), '') as varchar)) as _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid, - tmp.* -from __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab2 tmp --- simple_stream_with_namespace_resulting_into_long_names -)-- Final base SQL model -select - id, - date, - _airbyte_emitted_at, - _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid -from __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab3 --- simple_stream_with_namespace_resulting_into_long_names from "integrationtests".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql new file mode 100644 index 000000000000..c9bc7641794f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -0,0 +1,73 @@ + + + + create table + "integrationtests"."test_normalization"."nested_stream_with_complex_columns_resulting_into_long_names_scd__dbt_tmp" + + + compound sortkey(_airbyte_active_row,_airbyte_unique_key,_airbyte_emitted_at) + as ( + +with + +input_data as ( + select * + from "integrationtests"._airbyte_test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_ab3" + -- nested_stream_with_complex_columns_resulting_into_long_names from "integrationtests".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast(id as varchar), '') as varchar)) as _airbyte_unique_key, + id, + date, + "partition", + date as _airbyte_start_at, + lag(date) over ( + partition by id + order by + date is null asc, + date desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(date) over ( + partition by id + order by + date is null asc, + date desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(coalesce(cast(_airbyte_unique_key as varchar), '') || '-' || coalesce(cast(_airbyte_start_at as varchar), '') || '-' || coalesce(cast(_airbyte_emitted_at as varchar), '') as varchar)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + date, + "partition", + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid +from dedup_data where _airbyte_row_num = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql new file mode 100644 index 000000000000..0cb6fda50a97 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql @@ -0,0 +1,27 @@ + + + + create table + "integrationtests"."test_normalization"."nested_stream_with_complex_columns_resulting_into_long_names__dbt_tmp" + + + compound sortkey(_airbyte_unique_key,_airbyte_emitted_at) + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + date, + "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid +from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_scd" +-- nested_stream_with_complex_columns_resulting_into_long_names from "integrationtests".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 +and _airbyte_active_row = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..e6dd29796238 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql @@ -0,0 +1,62 @@ + + + + create table + "integrationtests"."test_normalization"."unnest_alias__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__unnest_alias_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + json_extract_path_text(_airbyte_data, 'children', true) as children, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_unnest_alias as table_alias +-- unnest_alias +where 1 = 1 + +), __dbt__cte__unnest_alias_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + children, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__unnest_alias_ab1 +-- unnest_alias +where 1 = 1 + +), __dbt__cte__unnest_alias_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(children as varchar), '') as varchar)) as _airbyte_unnest_alias_hashid, + tmp.* +from __dbt__cte__unnest_alias_ab2 tmp +-- unnest_alias +where 1 = 1 + +)-- Final base SQL model +select + id, + children, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_unnest_alias_hashid +from __dbt__cte__unnest_alias_ab3 +-- unnest_alias from "integrationtests".test_normalization._airbyte_raw_unnest_alias +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql new file mode 100644 index 000000000000..6a49f0ff964e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -0,0 +1,54 @@ + + + create table + "integrationtests".test_normalization."conflict_stream_array__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__conflict_stream_array_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + json_extract_path_text(_airbyte_data, 'conflict_stream_array', true) as conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_conflict_stream_array as table_alias +-- conflict_stream_array +where 1 = 1 +), __dbt__cte__conflict_stream_array_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as varchar) as id, + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__conflict_stream_array_ab1 +-- conflict_stream_array +where 1 = 1 +), __dbt__cte__conflict_stream_array_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(conflict_stream_array as varchar), '') as varchar)) as _airbyte_conflict_stream_array_hashid, + tmp.* +from __dbt__cte__conflict_stream_array_ab2 tmp +-- conflict_stream_array +where 1 = 1 +)-- Final base SQL model +select + id, + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_conflict_stream_array_hashid +from __dbt__cte__conflict_stream_array_ab3 +-- conflict_stream_array from "integrationtests".test_normalization._airbyte_raw_conflict_stream_array +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql new file mode 100644 index 000000000000..13aba9129a52 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -0,0 +1,56 @@ + + + create table + "integrationtests".test_normalization."conflict_stream_name__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__conflict_stream_name_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + + case when json_extract_path_text(table_alias._airbyte_data, 'conflict_stream_name', true) != '' then json_extract_path_text(table_alias._airbyte_data, 'conflict_stream_name', true) end + as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_conflict_stream_name as table_alias +-- conflict_stream_name +where 1 = 1 +), __dbt__cte__conflict_stream_name_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as varchar) as id, + cast(conflict_stream_name as varchar) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__conflict_stream_name_ab1 +-- conflict_stream_name +where 1 = 1 +), __dbt__cte__conflict_stream_name_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(conflict_stream_name as varchar), '') as varchar)) as _airbyte_conflict_stream_name_hashid, + tmp.* +from __dbt__cte__conflict_stream_name_ab2 tmp +-- conflict_stream_name +where 1 = 1 +)-- Final base SQL model +select + id, + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_hashid +from __dbt__cte__conflict_stream_name_ab3 +-- conflict_stream_name from "integrationtests".test_normalization._airbyte_raw_conflict_stream_name +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql new file mode 100644 index 000000000000..bab7f7d138d5 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -0,0 +1,57 @@ + + + create table + "integrationtests".test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__conflict_stream_name_conflict_stream_name_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_hashid, + + case when json_extract_path_text(table_alias.conflict_stream_name, 'conflict_stream_name', true) != '' then json_extract_path_text(table_alias.conflict_stream_name, 'conflict_stream_name', true) end + as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization."conflict_stream_name" as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null +), __dbt__cte__conflict_stream_name_conflict_stream_name_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_hashid, + cast(conflict_stream_name as varchar) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__conflict_stream_name_conflict_stream_name_ab1 +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +), __dbt__cte__conflict_stream_name_conflict_stream_name_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_conflict_stream_name_hashid as varchar), '') || '-' || coalesce(cast(conflict_stream_name as varchar), '') as varchar)) as _airbyte_conflict_stream_name_2_hashid, + tmp.* +from __dbt__cte__conflict_stream_name_conflict_stream_name_ab2 tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +)-- Final base SQL model +select + _airbyte_conflict_stream_name_hashid, + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_2_hashid +from __dbt__cte__conflict_stream_name_conflict_stream_name_ab3 +-- conflict_stream_name at conflict_stream_name/conflict_stream_name from "integrationtests".test_normalization."conflict_stream_name" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql new file mode 100644 index 000000000000..19ac73995aa9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql @@ -0,0 +1,55 @@ + + + create table + "integrationtests".test_normalization."conflict_stream_name_conflict_stream_name_conflict_stream_name__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_2_hashid, + case when json_extract_path_text(conflict_stream_name, 'groups', true) != '' then json_extract_path_text(conflict_stream_name, 'groups', true) end as groups, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization."conflict_stream_name_conflict_stream_name" as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null +), __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_2_hashid, + cast(groups as varchar) as groups, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1 +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +), __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as varchar), '') || '-' || coalesce(cast(groups as varchar), '') as varchar)) as _airbyte_conflict_stream_name_3_hashid, + tmp.* +from __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2 tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +)-- Final base SQL model +select + _airbyte_conflict_stream_name_2_hashid, + groups, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_3_hashid +from __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3 +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from "integrationtests".test_normalization."conflict_stream_name_conflict_stream_name" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql new file mode 100644 index 000000000000..bd6a7edaa276 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -0,0 +1,56 @@ + + + create table + "integrationtests".test_normalization."conflict_stream_scalar__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__conflict_stream_scalar_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'conflict_stream_scalar', true) != '' then json_extract_path_text(_airbyte_data, 'conflict_stream_scalar', true) end as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_conflict_stream_scalar as table_alias +-- conflict_stream_scalar +where 1 = 1 +), __dbt__cte__conflict_stream_scalar_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as varchar) as id, + cast(conflict_stream_scalar as + bigint +) as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__conflict_stream_scalar_ab1 +-- conflict_stream_scalar +where 1 = 1 +), __dbt__cte__conflict_stream_scalar_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(conflict_stream_scalar as varchar), '') as varchar)) as _airbyte_conflict_stream_scalar_hashid, + tmp.* +from __dbt__cte__conflict_stream_scalar_ab2 tmp +-- conflict_stream_scalar +where 1 = 1 +)-- Final base SQL model +select + id, + conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_conflict_stream_scalar_hashid +from __dbt__cte__conflict_stream_scalar_ab3 +-- conflict_stream_scalar from "integrationtests".test_normalization._airbyte_raw_conflict_stream_scalar +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql new file mode 100644 index 000000000000..f691cc5da8fb --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql @@ -0,0 +1,61 @@ + + + create table + "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + json_extract_path_text("partition", 'double_array_data', true) as double_array_data, + json_extract_path_text("partition", 'DATA', true) as data, + json_extract_path_text("partition", 'column`_''with"_quotes', true) as "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names" as table_alias +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and "partition" is not null +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + double_array_data, + data, + "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1 +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid as varchar), '') || '-' || coalesce(cast(double_array_data as varchar), '') || '-' || coalesce(cast(data as varchar), '') || '-' || coalesce(cast("column`_'with""_quotes" as varchar), '') as varchar)) as _airbyte_partition_hashid, + tmp.* +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2 tmp +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +)-- Final base SQL model +select + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + double_array_data, + data, + "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_partition_hashid +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3 +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql new file mode 100644 index 000000000000..9eb1e109077e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql @@ -0,0 +1,100 @@ + + + create table + "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * power(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 1 + order by generated_number + + +), +joined as ( + select + _airbyte_partition_hashid as _airbyte_hashid, + json_extract_array_element_text("column`_'with""_quotes", numbers.generated_number::int - 1, true) as _airbyte_nested_data + from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in nested_stream_with_complex_columns_resulting_into_long_names_partition."column`_'with""_quotes" + where numbers.generated_number <= json_array_length("column`_'with""_quotes", true) +) +select + _airbyte_partition_hashid, + case when json_extract_path_text(_airbyte_nested_data, 'currency', true) != '' then json_extract_path_text(_airbyte_nested_data, 'currency', true) end as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and "column`_'with""_quotes" is not null +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as varchar) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1 +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_partition_hashid as varchar), '') || '-' || coalesce(cast(currency as varchar), '') as varchar)) as _airbyte_column___with__quotes_hashid, + tmp.* +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2 tmp +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 +)-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_column___with__quotes_hashid +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3 +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql new file mode 100644 index 000000000000..ed667f0e4087 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql @@ -0,0 +1,100 @@ + + + create table + "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition_data__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * power(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 1 + order by generated_number + + +), +joined as ( + select + _airbyte_partition_hashid as _airbyte_hashid, + json_extract_array_element_text(data, numbers.generated_number::int - 1, true) as _airbyte_nested_data + from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in nested_stream_with_complex_columns_resulting_into_long_names_partition.data + where numbers.generated_number <= json_array_length(data, true) +) +select + _airbyte_partition_hashid, + case when json_extract_path_text(_airbyte_nested_data, 'currency', true) != '' then json_extract_path_text(_airbyte_nested_data, 'currency', true) end as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias +-- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and data is not null +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as varchar) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1 +-- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_partition_hashid as varchar), '') || '-' || coalesce(cast(currency as varchar), '') as varchar)) as _airbyte_data_hashid, + tmp.* +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2 tmp +-- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 +)-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_data_hashid +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3 +-- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql new file mode 100644 index 000000000000..51f7dcf22ca2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql @@ -0,0 +1,100 @@ + + + create table + "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * power(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 2 + order by generated_number + + +), +joined as ( + select + _airbyte_partition_hashid as _airbyte_hashid, + json_extract_array_element_text(double_array_data, numbers.generated_number::int - 1, true) as _airbyte_nested_data + from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in nested_stream_with_complex_columns_resulting_into_long_names_partition.double_array_data + where numbers.generated_number <= json_array_length(double_array_data, true) +) +select + _airbyte_partition_hashid, + case when json_extract_path_text(_airbyte_nested_data, 'id', true) != '' then json_extract_path_text(_airbyte_nested_data, 'id', true) end as id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and double_array_data is not null +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(id as varchar) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1 +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_partition_hashid as varchar), '') || '-' || coalesce(cast(id as varchar), '') as varchar)) as _airbyte_double_array_data_hashid, + tmp.* +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2 tmp +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 +)-- Final base SQL model +select + _airbyte_partition_hashid, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_double_array_data_hashid +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3 +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql new file mode 100644 index 000000000000..95f350d3abdd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql @@ -0,0 +1,54 @@ + + + create table + "integrationtests".test_normalization."non_nested_stream_without_namespace_resulting_into_long_names__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias +-- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 +), __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as varchar) as id, + cast(date as varchar) as date, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab1 +-- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 +), __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(date as varchar), '') as varchar)) as _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid, + tmp.* +from __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab2 tmp +-- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 +)-- Final base SQL model +select + id, + date, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid +from __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab3 +-- non_nested_stream_without_namespace_resulting_into_long_names from "integrationtests".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql new file mode 100644 index 000000000000..aca1269c8d5f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -0,0 +1,107 @@ + + + create table + "integrationtests".test_normalization."unnest_alias_children__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__unnest_alias_children_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * power(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 2 + order by generated_number + + +), +joined as ( + select + _airbyte_unnest_alias_hashid as _airbyte_hashid, + json_extract_array_element_text(children, numbers.generated_number::int - 1, true) as _airbyte_nested_data + from "integrationtests".test_normalization."unnest_alias" + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in unnest_alias.children + where numbers.generated_number <= json_array_length(children, true) +) +select + _airbyte_unnest_alias_hashid, + case when json_extract_path_text(_airbyte_nested_data, 'ab_id', true) != '' then json_extract_path_text(_airbyte_nested_data, 'ab_id', true) end as ab_id, + + case when json_extract_path_text(_airbyte_nested_data, 'owner', true) != '' then json_extract_path_text(_airbyte_nested_data, 'owner', true) end + as owner, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization."unnest_alias" as table_alias +-- children at unnest_alias/children +left join joined on _airbyte_unnest_alias_hashid = joined._airbyte_hashid +where 1 = 1 +and children is not null +), __dbt__cte__unnest_alias_children_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_unnest_alias_hashid, + cast(ab_id as + bigint +) as ab_id, + cast(owner as varchar) as owner, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__unnest_alias_children_ab1 +-- children at unnest_alias/children +where 1 = 1 +), __dbt__cte__unnest_alias_children_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_unnest_alias_hashid as varchar), '') || '-' || coalesce(cast(ab_id as varchar), '') || '-' || coalesce(cast(owner as varchar), '') as varchar)) as _airbyte_children_hashid, + tmp.* +from __dbt__cte__unnest_alias_children_ab2 tmp +-- children at unnest_alias/children +where 1 = 1 +)-- Final base SQL model +select + _airbyte_unnest_alias_hashid, + ab_id, + owner, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_children_hashid +from __dbt__cte__unnest_alias_children_ab3 +-- children at unnest_alias/children from "integrationtests".test_normalization."unnest_alias" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql new file mode 100644 index 000000000000..d6c293674dbe --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -0,0 +1,57 @@ + + + create table + "integrationtests".test_normalization."unnest_alias_children_owner__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__unnest_alias_children_owner_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_children_hashid, + case when json_extract_path_text(owner, 'owner_id', true) != '' then json_extract_path_text(owner, 'owner_id', true) end as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization."unnest_alias_children" as table_alias +-- owner at unnest_alias/children/owner +where 1 = 1 +and owner is not null +), __dbt__cte__unnest_alias_children_owner_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_children_hashid, + cast(owner_id as + bigint +) as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__unnest_alias_children_owner_ab1 +-- owner at unnest_alias/children/owner +where 1 = 1 +), __dbt__cte__unnest_alias_children_owner_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_children_hashid as varchar), '') || '-' || coalesce(cast(owner_id as varchar), '') as varchar)) as _airbyte_owner_hashid, + tmp.* +from __dbt__cte__unnest_alias_children_owner_ab2 tmp +-- owner at unnest_alias/children/owner +where 1 = 1 +)-- Final base SQL model +select + _airbyte_children_hashid, + owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_owner_hashid +from __dbt__cte__unnest_alias_children_owner_ab3 +-- owner at unnest_alias/children/owner from "integrationtests".test_normalization."unnest_alias_children" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql new file mode 100644 index 000000000000..69dff6931661 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql @@ -0,0 +1,54 @@ + + + create table + "integrationtests".test_normalization_namespace."simple_stream_with_namespace_resulting_into_long_names__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias +-- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 +), __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as varchar) as id, + cast(date as varchar) as date, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab1 +-- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 +), __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(date as varchar), '') as varchar)) as _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid, + tmp.* +from __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab2 tmp +-- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 +)-- Final base SQL model +select + id, + date, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid +from __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab3 +-- simple_stream_with_namespace_resulting_into_long_names from "integrationtests".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql new file mode 100644 index 000000000000..361919a65bc2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql @@ -0,0 +1,43 @@ + + + create view "integrationtests"._airbyte_test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_ab3__dbt_tmp" as ( + +with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, + + case when json_extract_path_text(table_alias._airbyte_data, 'partition', true) != '' then json_extract_path_text(table_alias._airbyte_data, 'partition', true) end + as "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 + +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as varchar) as id, + cast(date as varchar) as date, + cast("partition" as varchar) as "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab1 +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(date as varchar), '') || '-' || coalesce(cast("partition" as varchar), '') as varchar)) as _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + tmp.* +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab2 tmp +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 + + ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql deleted file mode 100644 index f08db80c0559..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql +++ /dev/null @@ -1,19 +0,0 @@ -with table_row_counts as ( - select distinct count(*) as row_count, 2 as expected_count - from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} -union all - select distinct count(*) as row_count, 2 as expected_count - from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names') }} -union all - select distinct count(*) as row_count, 2 as expected_count - from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} -union all - select count(distinct currency) as row_count, 1 as expected_count - from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_data') }} --- union all --- select count(distinct id) as row_count, 3 as expected_count --- from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data') }} -) -select * -from table_row_counts -where row_count != expected_count diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql index d3df56ab3d35..8338f7a9440b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_array('_airbyte_data', ['conflict_stream_array'], ['conflict_stream_array']) }} as conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} as table_alias -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql index 2f0da1bb06e7..47b4d187fc92 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_array_ab1') }} -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql index 0a92c655f92a..d4efd628548e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('conflict_stream_array_ab2') }} tmp -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql index 8556d7f50afb..223381444e1f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract('table_alias', '_airbyte_data', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} as table_alias -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql index a1d77f379bce..02a958942c9c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_ab1') }} -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql index dc562af89ba1..89a2bdb2f1ed 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('conflict_stream_name_ab2') }} tmp -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql index d6e2edb75ced..a8a342548c36 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql @@ -1,10 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_conflict_stream_name_hashid, {{ json_extract('table_alias', 'conflict_stream_name', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name') }} as table_alias -where conflict_stream_name is not null -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql index 463a878933f8..3c458a14224f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_conflict_stream_name_hashid, cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_conflict_stream_name_ab1') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql index f8d96123ce25..aa0d9c1cb20b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('conflict_stream_name_conflict_stream_name_ab2') }} tmp -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql index 77e7bd31c512..d69a8acbadec 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql @@ -1,10 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_conflict_stream_name_2_hashid, {{ json_extract_scalar('conflict_stream_name', ['groups'], ['groups']) }} as groups, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_conflict_stream_name') }} as table_alias -where conflict_stream_name is not null -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql index 50d9169b110d..dc826e7f53c2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_conflict_stream_name_2_hashid, cast(groups as {{ dbt_utils.type_string() }}) as groups, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql index 2f827f7347b8..9b7dfbfccab7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2') }} tmp -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql index be57970e399e..65a84615e748 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['conflict_stream_scalar'], ['conflict_stream_scalar']) }} as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} as table_alias -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql index cf0d9deb51cd..6205a85370ec 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast(conflict_stream_scalar as {{ dbt_utils.type_bigint() }}) as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_scalar_ab1') }} -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql index 781c4bbfc391..42f6c144d3c6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('conflict_stream_scalar_ab2') }} tmp -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql index aaa4eb63f243..58d09347649a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql @@ -1,10 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as date, {{ json_extract('table_alias', '_airbyte_data', ['partition'], ['partition']) }} as {{ adapter.quote('partition') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} as table_alias -- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql index 63bd774a716f..9369b7333ece 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql @@ -1,10 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast(date as {{ dbt_utils.type_string() }}) as date, cast({{ adapter.quote('partition') }} as {{ type_json() }}) as {{ adapter.quote('partition') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab1') }} -- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql deleted file mode 100644 index 23ffaa321bd8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql +++ /dev/null @@ -1,12 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'date', - adapter.quote('partition'), - ]) }} as _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, - tmp.* -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab2') }} tmp --- nested_stream_with_complex_columns_resulting_into_long_names - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab4.sql deleted file mode 100644 index e2c9a1304fde..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab3') }} tmp --- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql index fb632de52d62..fc17127ac094 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql @@ -1,12 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, {{ json_extract_array(adapter.quote('partition'), ['double_array_data'], ['double_array_data']) }} as double_array_data, {{ json_extract_array(adapter.quote('partition'), ['DATA'], ['DATA']) }} as data, {{ json_extract_array(adapter.quote('partition'), ['column`_\'with"_quotes'], ['column___with__quotes']) }} as {{ adapter.quote('column`_\'with""_quotes') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names') }} as table_alias -where {{ adapter.quote('partition') }} is not null -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and {{ adapter.quote('partition') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql index 7ade5b21f875..aa1074726da6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql @@ -1,11 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, double_array_data, data, {{ adapter.quote('column`_\'with""_quotes') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1') }} -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql index 0a034922317e..43e9f45bf5f1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -10,4 +15,5 @@ select tmp.* from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2') }} tmp -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql index 06114ee71d7e..f3478375aeaf 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql @@ -1,12 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_complex_columns_resulting_into_long_names_partition', 'partition', adapter.quote('column`_\'with""_quotes')) }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value(adapter.quote('column`_\'with""_quotes')), ['currency'], ['currency']) }} as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', adapter.quote('column`_\'with""_quotes')) }} -where {{ adapter.quote('column`_\'with""_quotes') }} is not null -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +{{ cross_join_unnest('partition', adapter.quote('column`_\'with""_quotes')) }} +where 1 = 1 +and {{ adapter.quote('column`_\'with""_quotes') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql index 076b4bfc975c..ae54378f3109 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(currency as {{ dbt_utils.type_string() }}) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1') }} -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql index d4d5028ad31d..0fe19cecbd7c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2') }} tmp -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1.sql index cfacb333734f..791d76b4f4e3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1.sql @@ -1,12 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_complex_columns_resulting_into_long_names_partition', 'partition', 'data') }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value('data'), ['currency'], ['currency']) }} as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', 'data') }} -where data is not null -- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +{{ cross_join_unnest('partition', 'data') }} +where 1 = 1 +and data is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2.sql index b4092b9210a8..01ca205eece7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(currency as {{ dbt_utils.type_string() }}) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1') }} -- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3.sql index 69c04aca4634..377ca6ed0320 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2') }} tmp -- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql index 48b7ea236ae1..5268dd0dbd4f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql @@ -1,12 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_complex_columns_resulting_into_long_names_partition', 'partition', 'double_array_data') }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value('double_array_data'), ['id'], ['id']) }} as id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', 'double_array_data') }} -where double_array_data is not null -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +{{ cross_join_unnest('partition', 'double_array_data') }} +where 1 = 1 +and double_array_data is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql index a3e29e92dfb7..cb8eb9675598 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(id as {{ dbt_utils.type_string() }}) as id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1') }} -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql index 13295b47f97f..3972432d4691 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2') }} tmp -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql index fd2fbdd54863..a80e0a1c756b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as date, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names') }} as table_alias -- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql index e4ed8d075d22..a8aee99469e5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast(date as {{ dbt_utils.type_string() }}) as date, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('non_nested_stream_without_namespace_resulting_into_long_names_ab1') }} -- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql index cd80649af632..f55f72315f8a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('non_nested_stream_without_namespace_resulting_into_long_names_ab2') }} tmp -- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql index 15b34c1ca7d4..d4f394ed7911 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_array('_airbyte_data', ['children'], ['children']) }} as children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} as table_alias -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql index a416538b5422..2ccb9349d718 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_ab1') }} -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql index 0c6703933b10..68c9c6064d65 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,6 @@ select tmp.* from {{ ref('unnest_alias_ab2') }} tmp -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql index 57abeb23159f..1a67101d4ea8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql @@ -1,13 +1,21 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('unnest_alias', 'unnest_alias', 'children') }} select _airbyte_unnest_alias_hashid, {{ json_extract_scalar(unnested_column_value('children'), ['ab_id'], ['ab_id']) }} as ab_id, {{ json_extract('', unnested_column_value('children'), ['owner'], ['owner']) }} as owner, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias') }} as table_alias -{{ cross_join_unnest('unnest_alias', 'children') }} -where children is not null -- children at unnest_alias/children +{{ cross_join_unnest('unnest_alias', 'children') }} +where 1 = 1 +and children is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql index 8db853c1d601..4c4c2367a5f6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql @@ -1,10 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_unnest_alias_hashid, cast(ab_id as {{ dbt_utils.type_bigint() }}) as ab_id, cast(owner as {{ type_json() }}) as owner, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children_ab1') }} -- children at unnest_alias/children +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql index 9d84f3be6aa0..06e2a4e3a52e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -9,4 +14,5 @@ select tmp.* from {{ ref('unnest_alias_children_ab2') }} tmp -- children at unnest_alias/children +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql index 4f28c4f8b5cb..0ed42cb28690 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql @@ -1,10 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_children_hashid, {{ json_extract_scalar('owner', ['owner_id'], ['owner_id']) }} as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children') }} as table_alias -where owner is not null -- owner at unnest_alias/children/owner +where 1 = 1 +and owner is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql index a0f474c62db7..f6b016bf5e78 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_children_hashid, cast(owner_id as {{ dbt_utils.type_bigint() }}) as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children_owner_ab1') }} -- owner at unnest_alias/children/owner +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql index c5f6562c34c2..1a77db63b778 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('unnest_alias_children_owner_ab2') }} tmp -- owner at unnest_alias/children/owner +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql index e3ec7ffd2f82..7ae7824eab31 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as date, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization_namespace', '_airbyte_raw_simple_stream_with_namespace_resulting_into_long_names') }} as table_alias -- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql index c437ca698089..3f8824570d2d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast(date as {{ dbt_utils.type_string() }}) as date, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('simple_stream_with_namespace_resulting_into_long_names_ab1') }} -- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql index 727eabab6ac2..0b1c938f2e3e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('simple_stream_with_namespace_resulting_into_long_names_ab2') }} tmp -- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql new file mode 100644 index 000000000000..80606e29b92c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -0,0 +1,108 @@ +{{ config( + sort = ["_airbyte_active_row", "_airbyte_unique_key", "_airbyte_emitted_at"], + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab3') }} + -- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('nested_stream_with_complex_columns_resulting_into_long_names_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('nested_stream_with_complex_columns_resulting_into_long_names_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('nested_stream_with_complex_columns_resulting_into_long_names_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab3') }} + -- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + date, + {{ adapter.quote('partition') }}, + date as _airbyte_start_at, + lag(date) over ( + partition by id + order by + date is null asc, + date desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(date) over ( + partition by id + order by + date is null asc, + date desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + date, + {{ adapter.quote('partition') }}, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql new file mode 100644 index 000000000000..06d33b00087f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql @@ -0,0 +1,22 @@ +{{ config( + sort = ["_airbyte_unique_key", "_airbyte_emitted_at"], + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + id, + date, + {{ adapter.quote('partition') }}, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid +from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_scd') }} +-- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..10f4b807149e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql @@ -0,0 +1,19 @@ +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + id, + children, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_unnest_alias_hashid +from {{ ref('unnest_alias_ab3') }} +-- unnest_alias from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql index 9ffda6086e35..328153a742fe 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, conflict_stream_array, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_array_hashid from {{ ref('conflict_stream_array_ab3') }} -- conflict_stream_array from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql index 8a83ccc01e31..51af86be3918 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_hashid from {{ ref('conflict_stream_name_ab3') }} -- conflict_stream_name from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql index 94a9dc3d0139..df29869e1b30 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_conflict_stream_name_hashid, conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_2_hashid from {{ ref('conflict_stream_name_conflict_stream_name_ab3') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql index 1a5bd998fa99..78325474c27a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_conflict_stream_name_2_hashid, groups, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_3_hashid from {{ ref('conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name_conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql index 9d7acc846fd0..e00b45e8686a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, conflict_stream_scalar, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_scalar_hashid from {{ ref('conflict_stream_scalar_ab3') }} -- conflict_stream_scalar from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql deleted file mode 100644 index 45aafa818f6f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql +++ /dev/null @@ -1,12 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - date, - {{ adapter.quote('partition') }}, - _airbyte_emitted_at, - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_scd') }} --- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} -where _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql index 8e228ffad1ab..6e3926b47284 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql @@ -1,12 +1,20 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, double_array_data, data, {{ adapter.quote('column`_\'with""_quotes') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_partition_hashid from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3') }} -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql index 2bd228318b0d..e8d0f08669f7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_column___with__quotes_hashid from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3') }} -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql index bdcfcf86eb2e..2186017e2c85 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_data_hashid from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3') }} -- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql index 50b39ee05b38..7a2ff26eb0f0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, id, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_double_array_data_hashid from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3') }} -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql deleted file mode 100644 index 06311da05fc6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - date, - {{ adapter.quote('partition') }}, - date as _airbyte_start_at, - lag(date) over ( - partition by id - order by date is null asc, date desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(date) over ( - partition by id - order by date is null asc, date desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab4') }} --- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql index 1004a9e2f4a4..c106d56f953c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, date, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid from {{ ref('non_nested_stream_without_namespace_resulting_into_long_names_ab3') }} -- non_nested_stream_without_namespace_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql deleted file mode 100644 index 7a570c3836cb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,10 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - children, - _airbyte_emitted_at, - _airbyte_unnest_alias_hashid -from {{ ref('unnest_alias_ab3') }} --- unnest_alias from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql index 77b2b0ad854e..9c5ef61b6354 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -1,11 +1,19 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_unnest_alias_hashid, ab_id, owner, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_children_hashid from {{ ref('unnest_alias_children_ab3') }} -- children at unnest_alias/children from {{ ref('unnest_alias') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql index 0ccf8e33d8d6..322eb9d5f0f6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_children_hashid, owner_id, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_owner_hashid from {{ ref('unnest_alias_children_owner_ab3') }} -- owner at unnest_alias/children/owner from {{ ref('unnest_alias_children') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql index ce74645f7864..08f4dde7545a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization_namespace", tags=["top-level"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization_namespace", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, date, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid from {{ ref('simple_stream_with_namespace_resulting_into_long_names_ab3') }} -- simple_stream_with_namespace_resulting_into_long_names from {{ source('test_normalization_namespace', '_airbyte_raw_simple_stream_with_namespace_resulting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql new file mode 100644 index 000000000000..aaf10fa8e809 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql @@ -0,0 +1,19 @@ +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + 'date', + adapter.quote('partition'), + ]) }} as _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + tmp.* +from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab2') }} tmp +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql new file mode 100644 index 000000000000..b3b47010b2a0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "nested_stream_with_complex_columns_resulti__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "date", "partition", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "date", "partition", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid" + from "nested_stream_with_complex_columns_resulti__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql new file mode 100644 index 000000000000..d307b3232ef7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "nested_stream_with_complex_columns_resulti__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names" ("_airbyte_unique_key", "id", "date", "partition", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid") + ( + select "_airbyte_unique_key", "id", "date", "partition", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid" + from "nested_stream_with_complex_columns_resulti__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..3c3524996abe --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."unnest_alias" + where (_airbyte_ab_id) in ( + select (_airbyte_ab_id) + from "unnest_alias__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."unnest_alias" ("id", "children", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_unnest_alias_hashid") + ( + select "id", "children", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_unnest_alias_hashid" + from "unnest_alias__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql new file mode 100644 index 000000000000..6a49f0ff964e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -0,0 +1,54 @@ + + + create table + "integrationtests".test_normalization."conflict_stream_array__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__conflict_stream_array_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + json_extract_path_text(_airbyte_data, 'conflict_stream_array', true) as conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_conflict_stream_array as table_alias +-- conflict_stream_array +where 1 = 1 +), __dbt__cte__conflict_stream_array_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as varchar) as id, + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__conflict_stream_array_ab1 +-- conflict_stream_array +where 1 = 1 +), __dbt__cte__conflict_stream_array_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(conflict_stream_array as varchar), '') as varchar)) as _airbyte_conflict_stream_array_hashid, + tmp.* +from __dbt__cte__conflict_stream_array_ab2 tmp +-- conflict_stream_array +where 1 = 1 +)-- Final base SQL model +select + id, + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_conflict_stream_array_hashid +from __dbt__cte__conflict_stream_array_ab3 +-- conflict_stream_array from "integrationtests".test_normalization._airbyte_raw_conflict_stream_array +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql new file mode 100644 index 000000000000..13aba9129a52 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -0,0 +1,56 @@ + + + create table + "integrationtests".test_normalization."conflict_stream_name__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__conflict_stream_name_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + + case when json_extract_path_text(table_alias._airbyte_data, 'conflict_stream_name', true) != '' then json_extract_path_text(table_alias._airbyte_data, 'conflict_stream_name', true) end + as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_conflict_stream_name as table_alias +-- conflict_stream_name +where 1 = 1 +), __dbt__cte__conflict_stream_name_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as varchar) as id, + cast(conflict_stream_name as varchar) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__conflict_stream_name_ab1 +-- conflict_stream_name +where 1 = 1 +), __dbt__cte__conflict_stream_name_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(conflict_stream_name as varchar), '') as varchar)) as _airbyte_conflict_stream_name_hashid, + tmp.* +from __dbt__cte__conflict_stream_name_ab2 tmp +-- conflict_stream_name +where 1 = 1 +)-- Final base SQL model +select + id, + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_hashid +from __dbt__cte__conflict_stream_name_ab3 +-- conflict_stream_name from "integrationtests".test_normalization._airbyte_raw_conflict_stream_name +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql new file mode 100644 index 000000000000..bab7f7d138d5 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -0,0 +1,57 @@ + + + create table + "integrationtests".test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__conflict_stream_name_conflict_stream_name_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_hashid, + + case when json_extract_path_text(table_alias.conflict_stream_name, 'conflict_stream_name', true) != '' then json_extract_path_text(table_alias.conflict_stream_name, 'conflict_stream_name', true) end + as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization."conflict_stream_name" as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null +), __dbt__cte__conflict_stream_name_conflict_stream_name_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_hashid, + cast(conflict_stream_name as varchar) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__conflict_stream_name_conflict_stream_name_ab1 +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +), __dbt__cte__conflict_stream_name_conflict_stream_name_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_conflict_stream_name_hashid as varchar), '') || '-' || coalesce(cast(conflict_stream_name as varchar), '') as varchar)) as _airbyte_conflict_stream_name_2_hashid, + tmp.* +from __dbt__cte__conflict_stream_name_conflict_stream_name_ab2 tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +)-- Final base SQL model +select + _airbyte_conflict_stream_name_hashid, + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_2_hashid +from __dbt__cte__conflict_stream_name_conflict_stream_name_ab3 +-- conflict_stream_name at conflict_stream_name/conflict_stream_name from "integrationtests".test_normalization."conflict_stream_name" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql new file mode 100644 index 000000000000..19ac73995aa9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql @@ -0,0 +1,55 @@ + + + create table + "integrationtests".test_normalization."conflict_stream_name_conflict_stream_name_conflict_stream_name__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_2_hashid, + case when json_extract_path_text(conflict_stream_name, 'groups', true) != '' then json_extract_path_text(conflict_stream_name, 'groups', true) end as groups, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization."conflict_stream_name_conflict_stream_name" as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null +), __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_2_hashid, + cast(groups as varchar) as groups, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1 +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +), __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as varchar), '') || '-' || coalesce(cast(groups as varchar), '') as varchar)) as _airbyte_conflict_stream_name_3_hashid, + tmp.* +from __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2 tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +)-- Final base SQL model +select + _airbyte_conflict_stream_name_2_hashid, + groups, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_3_hashid +from __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3 +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from "integrationtests".test_normalization."conflict_stream_name_conflict_stream_name" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql new file mode 100644 index 000000000000..bd6a7edaa276 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -0,0 +1,56 @@ + + + create table + "integrationtests".test_normalization."conflict_stream_scalar__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__conflict_stream_scalar_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'conflict_stream_scalar', true) != '' then json_extract_path_text(_airbyte_data, 'conflict_stream_scalar', true) end as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_conflict_stream_scalar as table_alias +-- conflict_stream_scalar +where 1 = 1 +), __dbt__cte__conflict_stream_scalar_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as varchar) as id, + cast(conflict_stream_scalar as + bigint +) as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__conflict_stream_scalar_ab1 +-- conflict_stream_scalar +where 1 = 1 +), __dbt__cte__conflict_stream_scalar_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(conflict_stream_scalar as varchar), '') as varchar)) as _airbyte_conflict_stream_scalar_hashid, + tmp.* +from __dbt__cte__conflict_stream_scalar_ab2 tmp +-- conflict_stream_scalar +where 1 = 1 +)-- Final base SQL model +select + id, + conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_conflict_stream_scalar_hashid +from __dbt__cte__conflict_stream_scalar_ab3 +-- conflict_stream_scalar from "integrationtests".test_normalization._airbyte_raw_conflict_stream_scalar +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql new file mode 100644 index 000000000000..f691cc5da8fb --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql @@ -0,0 +1,61 @@ + + + create table + "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + json_extract_path_text("partition", 'double_array_data', true) as double_array_data, + json_extract_path_text("partition", 'DATA', true) as data, + json_extract_path_text("partition", 'column`_''with"_quotes', true) as "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names" as table_alias +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and "partition" is not null +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + double_array_data, + data, + "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1 +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid as varchar), '') || '-' || coalesce(cast(double_array_data as varchar), '') || '-' || coalesce(cast(data as varchar), '') || '-' || coalesce(cast("column`_'with""_quotes" as varchar), '') as varchar)) as _airbyte_partition_hashid, + tmp.* +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2 tmp +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +)-- Final base SQL model +select + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + double_array_data, + data, + "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_partition_hashid +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3 +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql new file mode 100644 index 000000000000..9eb1e109077e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql @@ -0,0 +1,100 @@ + + + create table + "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * power(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 1 + order by generated_number + + +), +joined as ( + select + _airbyte_partition_hashid as _airbyte_hashid, + json_extract_array_element_text("column`_'with""_quotes", numbers.generated_number::int - 1, true) as _airbyte_nested_data + from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in nested_stream_with_complex_columns_resulting_into_long_names_partition."column`_'with""_quotes" + where numbers.generated_number <= json_array_length("column`_'with""_quotes", true) +) +select + _airbyte_partition_hashid, + case when json_extract_path_text(_airbyte_nested_data, 'currency', true) != '' then json_extract_path_text(_airbyte_nested_data, 'currency', true) end as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and "column`_'with""_quotes" is not null +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as varchar) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1 +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_partition_hashid as varchar), '') || '-' || coalesce(cast(currency as varchar), '') as varchar)) as _airbyte_column___with__quotes_hashid, + tmp.* +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2 tmp +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 +)-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_column___with__quotes_hashid +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3 +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql new file mode 100644 index 000000000000..ed667f0e4087 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql @@ -0,0 +1,100 @@ + + + create table + "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition_data__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * power(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 1 + order by generated_number + + +), +joined as ( + select + _airbyte_partition_hashid as _airbyte_hashid, + json_extract_array_element_text(data, numbers.generated_number::int - 1, true) as _airbyte_nested_data + from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in nested_stream_with_complex_columns_resulting_into_long_names_partition.data + where numbers.generated_number <= json_array_length(data, true) +) +select + _airbyte_partition_hashid, + case when json_extract_path_text(_airbyte_nested_data, 'currency', true) != '' then json_extract_path_text(_airbyte_nested_data, 'currency', true) end as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias +-- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and data is not null +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as varchar) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1 +-- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_partition_hashid as varchar), '') || '-' || coalesce(cast(currency as varchar), '') as varchar)) as _airbyte_data_hashid, + tmp.* +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2 tmp +-- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 +)-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_data_hashid +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3 +-- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql new file mode 100644 index 000000000000..51f7dcf22ca2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql @@ -0,0 +1,100 @@ + + + create table + "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * power(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 2 + order by generated_number + + +), +joined as ( + select + _airbyte_partition_hashid as _airbyte_hashid, + json_extract_array_element_text(double_array_data, numbers.generated_number::int - 1, true) as _airbyte_nested_data + from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in nested_stream_with_complex_columns_resulting_into_long_names_partition.double_array_data + where numbers.generated_number <= json_array_length(double_array_data, true) +) +select + _airbyte_partition_hashid, + case when json_extract_path_text(_airbyte_nested_data, 'id', true) != '' then json_extract_path_text(_airbyte_nested_data, 'id', true) end as id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and double_array_data is not null +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(id as varchar) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1 +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_partition_hashid as varchar), '') || '-' || coalesce(cast(id as varchar), '') as varchar)) as _airbyte_double_array_data_hashid, + tmp.* +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2 tmp +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 +)-- Final base SQL model +select + _airbyte_partition_hashid, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_double_array_data_hashid +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3 +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql new file mode 100644 index 000000000000..95f350d3abdd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql @@ -0,0 +1,54 @@ + + + create table + "integrationtests".test_normalization."non_nested_stream_without_namespace_resulting_into_long_names__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias +-- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 +), __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as varchar) as id, + cast(date as varchar) as date, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab1 +-- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 +), __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(date as varchar), '') as varchar)) as _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid, + tmp.* +from __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab2 tmp +-- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 +)-- Final base SQL model +select + id, + date, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid +from __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab3 +-- non_nested_stream_without_namespace_resulting_into_long_names from "integrationtests".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql new file mode 100644 index 000000000000..aca1269c8d5f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -0,0 +1,107 @@ + + + create table + "integrationtests".test_normalization."unnest_alias_children__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__unnest_alias_children_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * power(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 2 + order by generated_number + + +), +joined as ( + select + _airbyte_unnest_alias_hashid as _airbyte_hashid, + json_extract_array_element_text(children, numbers.generated_number::int - 1, true) as _airbyte_nested_data + from "integrationtests".test_normalization."unnest_alias" + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in unnest_alias.children + where numbers.generated_number <= json_array_length(children, true) +) +select + _airbyte_unnest_alias_hashid, + case when json_extract_path_text(_airbyte_nested_data, 'ab_id', true) != '' then json_extract_path_text(_airbyte_nested_data, 'ab_id', true) end as ab_id, + + case when json_extract_path_text(_airbyte_nested_data, 'owner', true) != '' then json_extract_path_text(_airbyte_nested_data, 'owner', true) end + as owner, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization."unnest_alias" as table_alias +-- children at unnest_alias/children +left join joined on _airbyte_unnest_alias_hashid = joined._airbyte_hashid +where 1 = 1 +and children is not null +), __dbt__cte__unnest_alias_children_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_unnest_alias_hashid, + cast(ab_id as + bigint +) as ab_id, + cast(owner as varchar) as owner, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__unnest_alias_children_ab1 +-- children at unnest_alias/children +where 1 = 1 +), __dbt__cte__unnest_alias_children_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_unnest_alias_hashid as varchar), '') || '-' || coalesce(cast(ab_id as varchar), '') || '-' || coalesce(cast(owner as varchar), '') as varchar)) as _airbyte_children_hashid, + tmp.* +from __dbt__cte__unnest_alias_children_ab2 tmp +-- children at unnest_alias/children +where 1 = 1 +)-- Final base SQL model +select + _airbyte_unnest_alias_hashid, + ab_id, + owner, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_children_hashid +from __dbt__cte__unnest_alias_children_ab3 +-- children at unnest_alias/children from "integrationtests".test_normalization."unnest_alias" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql new file mode 100644 index 000000000000..d6c293674dbe --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -0,0 +1,57 @@ + + + create table + "integrationtests".test_normalization."unnest_alias_children_owner__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__unnest_alias_children_owner_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_children_hashid, + case when json_extract_path_text(owner, 'owner_id', true) != '' then json_extract_path_text(owner, 'owner_id', true) end as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization."unnest_alias_children" as table_alias +-- owner at unnest_alias/children/owner +where 1 = 1 +and owner is not null +), __dbt__cte__unnest_alias_children_owner_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_children_hashid, + cast(owner_id as + bigint +) as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__unnest_alias_children_owner_ab1 +-- owner at unnest_alias/children/owner +where 1 = 1 +), __dbt__cte__unnest_alias_children_owner_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_children_hashid as varchar), '') || '-' || coalesce(cast(owner_id as varchar), '') as varchar)) as _airbyte_owner_hashid, + tmp.* +from __dbt__cte__unnest_alias_children_owner_ab2 tmp +-- owner at unnest_alias/children/owner +where 1 = 1 +)-- Final base SQL model +select + _airbyte_children_hashid, + owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_owner_hashid +from __dbt__cte__unnest_alias_children_owner_ab3 +-- owner at unnest_alias/children/owner from "integrationtests".test_normalization."unnest_alias_children" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql new file mode 100644 index 000000000000..69dff6931661 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql @@ -0,0 +1,54 @@ + + + create table + "integrationtests".test_normalization_namespace."simple_stream_with_namespace_resulting_into_long_names__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias +-- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 +), __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as varchar) as id, + cast(date as varchar) as date, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab1 +-- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 +), __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(date as varchar), '') as varchar)) as _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid, + tmp.* +from __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab2 tmp +-- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 +)-- Final base SQL model +select + id, + date, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid +from __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab3 +-- simple_stream_with_namespace_resulting_into_long_names from "integrationtests".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql new file mode 100644 index 000000000000..361919a65bc2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql @@ -0,0 +1,43 @@ + + + create view "integrationtests"._airbyte_test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_ab3__dbt_tmp" as ( + +with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, + + case when json_extract_path_text(table_alias._airbyte_data, 'partition', true) != '' then json_extract_path_text(table_alias._airbyte_data, 'partition', true) end + as "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 + +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as varchar) as id, + cast(date as varchar) as date, + cast("partition" as varchar) as "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab1 +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(date as varchar), '') || '-' || coalesce(cast("partition" as varchar), '') as varchar)) as _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + tmp.* +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab2 tmp +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 + + ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index 9c685fb4b9b3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,22 +0,0 @@ - - - create table - "integrationtests".test_normalization."dedup_cdc_excluded__dbt_tmp" - - - as ( - --- Final base SQL model -select - id, - name, - "column`_'with""_quotes", - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from "integrationtests".test_normalization."dedup_cdc_excluded_scd" --- dedup_cdc_excluded from "integrationtests".test_normalization._airbyte_raw_dedup_cdc_excluded -where _airbyte_active_row = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index e0924e6bbf13..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,84 +0,0 @@ - - - create table - "integrationtests".test_normalization."dedup_cdc_excluded_scd__dbt_tmp" - - - as ( - -with __dbt__cte__dedup_cdc_excluded_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - case when json_extract_path_text(_airbyte_data, 'name', true) != '' then json_extract_path_text(_airbyte_data, 'name', true) end as name, - case when json_extract_path_text(_airbyte_data, 'column`_''with"_quotes', true) != '' then json_extract_path_text(_airbyte_data, 'column`_''with"_quotes', true) end as "column`_'with""_quotes", - case when json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) end as _ab_cdc_lsn, - case when json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) end as _ab_cdc_updated_at, - case when json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) end as _ab_cdc_deleted_at, - _airbyte_emitted_at -from "integrationtests".test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias --- dedup_cdc_excluded -), __dbt__cte__dedup_cdc_excluded_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - cast(name as varchar) as name, - cast("column`_'with""_quotes" as varchar) as "column`_'with""_quotes", - cast(_ab_cdc_lsn as - float -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float -) as _ab_cdc_deleted_at, - _airbyte_emitted_at -from __dbt__cte__dedup_cdc_excluded_ab1 --- dedup_cdc_excluded -), __dbt__cte__dedup_cdc_excluded_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(name as varchar), '') || '-' || coalesce(cast("column`_'with""_quotes" as varchar), '') || '-' || coalesce(cast(_ab_cdc_lsn as varchar), '') || '-' || coalesce(cast(_ab_cdc_updated_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as varchar), '') as varchar)) as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from __dbt__cte__dedup_cdc_excluded_ab2 tmp --- dedup_cdc_excluded -), __dbt__cte__dedup_cdc_excluded_ab4 as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_cdc_excluded_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from __dbt__cte__dedup_cdc_excluded_ab3 tmp --- dedup_cdc_excluded from "integrationtests".test_normalization._airbyte_raw_dedup_cdc_excluded -)-- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - "column`_'with""_quotes", - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from __dbt__cte__dedup_cdc_excluded_ab4 --- dedup_cdc_excluded from "integrationtests".test_normalization._airbyte_raw_dedup_cdc_excluded -where _airbyte_row_num = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql deleted file mode 100644 index 550f24e063f0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql +++ /dev/null @@ -1,24 +0,0 @@ - - - create table - "integrationtests".test_normalization."dedup_exchange_rate__dbt_tmp" - - - as ( - --- Final base SQL model -select - id, - currency, - date, - timestamp_col, - "hkd@spéçiäl & characters", - hkd_special___characters, - nzd, - usd, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from "integrationtests".test_normalization."dedup_exchange_rate_scd" --- dedup_exchange_rate from "integrationtests".test_normalization._airbyte_raw_dedup_exchange_rate -where _airbyte_active_row = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql deleted file mode 100644 index 96fdd13d575e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql +++ /dev/null @@ -1,94 +0,0 @@ - - - create table - "integrationtests".test_normalization."dedup_exchange_rate_scd__dbt_tmp" - - - as ( - -with __dbt__cte__dedup_exchange_rate_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - case when json_extract_path_text(_airbyte_data, 'currency', true) != '' then json_extract_path_text(_airbyte_data, 'currency', true) end as currency, - case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, - case when json_extract_path_text(_airbyte_data, 'timestamp_col', true) != '' then json_extract_path_text(_airbyte_data, 'timestamp_col', true) end as timestamp_col, - case when json_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters', true) != '' then json_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters', true) end as "hkd@spéçiäl & characters", - case when json_extract_path_text(_airbyte_data, 'HKD_special___characters', true) != '' then json_extract_path_text(_airbyte_data, 'HKD_special___characters', true) end as hkd_special___characters, - case when json_extract_path_text(_airbyte_data, 'NZD', true) != '' then json_extract_path_text(_airbyte_data, 'NZD', true) end as nzd, - case when json_extract_path_text(_airbyte_data, 'USD', true) != '' then json_extract_path_text(_airbyte_data, 'USD', true) end as usd, - _airbyte_emitted_at -from "integrationtests".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias --- dedup_exchange_rate -), __dbt__cte__dedup_exchange_rate_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - cast(currency as varchar) as currency, - cast(nullif(date, '') as - date -) as date, - cast(nullif(timestamp_col, '') as - timestamp with time zone -) as timestamp_col, - cast("hkd@spéçiäl & characters" as - float -) as "hkd@spéçiäl & characters", - cast(hkd_special___characters as varchar) as hkd_special___characters, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - _airbyte_emitted_at -from __dbt__cte__dedup_exchange_rate_ab1 --- dedup_exchange_rate -), __dbt__cte__dedup_exchange_rate_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(currency as varchar), '') || '-' || coalesce(cast(date as varchar), '') || '-' || coalesce(cast(timestamp_col as varchar), '') || '-' || coalesce(cast("hkd@spéçiäl & characters" as varchar), '') || '-' || coalesce(cast(hkd_special___characters as varchar), '') || '-' || coalesce(cast(nzd as varchar), '') || '-' || coalesce(cast(usd as varchar), '') as varchar)) as _airbyte_dedup_exchange_rate_hashid, - tmp.* -from __dbt__cte__dedup_exchange_rate_ab2 tmp --- dedup_exchange_rate -), __dbt__cte__dedup_exchange_rate_ab4 as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_exchange_rate_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from __dbt__cte__dedup_exchange_rate_ab3 tmp --- dedup_exchange_rate from "integrationtests".test_normalization._airbyte_raw_dedup_exchange_rate -)-- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - currency, - date, - timestamp_col, - "hkd@spéçiäl & characters", - hkd_special___characters, - nzd, - usd, - date as _airbyte_start_at, - lag(date) over ( - partition by id, currency, cast(nzd as varchar) - order by date is null asc, date desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(date) over ( - partition by id, currency, cast(nzd as varchar) - order by date is null asc, date desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from __dbt__cte__dedup_exchange_rate_ab4 --- dedup_exchange_rate from "integrationtests".test_normalization._airbyte_raw_dedup_exchange_rate -where _airbyte_row_num = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql deleted file mode 100644 index e0cab59bb19e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,73 +0,0 @@ - - - create table - "integrationtests".test_normalization."exchange_rate__dbt_tmp" - - - as ( - -with __dbt__cte__exchange_rate_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - case when json_extract_path_text(_airbyte_data, 'currency', true) != '' then json_extract_path_text(_airbyte_data, 'currency', true) end as currency, - case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, - case when json_extract_path_text(_airbyte_data, 'timestamp_col', true) != '' then json_extract_path_text(_airbyte_data, 'timestamp_col', true) end as timestamp_col, - case when json_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters', true) != '' then json_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters', true) end as "hkd@spéçiäl & characters", - case when json_extract_path_text(_airbyte_data, 'HKD_special___characters', true) != '' then json_extract_path_text(_airbyte_data, 'HKD_special___characters', true) end as hkd_special___characters, - case when json_extract_path_text(_airbyte_data, 'NZD', true) != '' then json_extract_path_text(_airbyte_data, 'NZD', true) end as nzd, - case when json_extract_path_text(_airbyte_data, 'USD', true) != '' then json_extract_path_text(_airbyte_data, 'USD', true) end as usd, - _airbyte_emitted_at -from "integrationtests".test_normalization._airbyte_raw_exchange_rate as table_alias --- exchange_rate -), __dbt__cte__exchange_rate_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - cast(currency as varchar) as currency, - cast(nullif(date, '') as - date -) as date, - cast(nullif(timestamp_col, '') as - timestamp with time zone -) as timestamp_col, - cast("hkd@spéçiäl & characters" as - float -) as "hkd@spéçiäl & characters", - cast(hkd_special___characters as varchar) as hkd_special___characters, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - _airbyte_emitted_at -from __dbt__cte__exchange_rate_ab1 --- exchange_rate -), __dbt__cte__exchange_rate_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(currency as varchar), '') || '-' || coalesce(cast(date as varchar), '') || '-' || coalesce(cast(timestamp_col as varchar), '') || '-' || coalesce(cast("hkd@spéçiäl & characters" as varchar), '') || '-' || coalesce(cast(hkd_special___characters as varchar), '') || '-' || coalesce(cast(nzd as varchar), '') || '-' || coalesce(cast(usd as varchar), '') as varchar)) as _airbyte_exchange_rate_hashid, - tmp.* -from __dbt__cte__exchange_rate_ab2 tmp --- exchange_rate -)-- Final base SQL model -select - id, - currency, - date, - timestamp_col, - "hkd@spéçiäl & characters", - hkd_special___characters, - nzd, - usd, - _airbyte_emitted_at, - _airbyte_exchange_rate_hashid -from __dbt__cte__exchange_rate_ab3 --- exchange_rate from "integrationtests".test_normalization._airbyte_raw_exchange_rate - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql deleted file mode 100644 index 1ed954b4633c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ /dev/null @@ -1,22 +0,0 @@ - - - create table - "integrationtests".test_normalization."pos_dedup_cdcx__dbt_tmp" - - - as ( - --- Final base SQL model -select - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid -from "integrationtests".test_normalization."pos_dedup_cdcx_scd" --- pos_dedup_cdcx from "integrationtests".test_normalization._airbyte_raw_pos_dedup_cdcx -where _airbyte_active_row = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index 3871ad68a7f4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,86 +0,0 @@ - - - create table - "integrationtests".test_normalization."pos_dedup_cdcx_scd__dbt_tmp" - - - as ( - -with __dbt__cte__pos_dedup_cdcx_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - case when json_extract_path_text(_airbyte_data, 'name', true) != '' then json_extract_path_text(_airbyte_data, 'name', true) end as name, - case when json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) end as _ab_cdc_lsn, - case when json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) end as _ab_cdc_updated_at, - case when json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) end as _ab_cdc_deleted_at, - case when json_extract_path_text(_airbyte_data, '_ab_cdc_log_pos', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_log_pos', true) end as _ab_cdc_log_pos, - _airbyte_emitted_at -from "integrationtests".test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias --- pos_dedup_cdcx -), __dbt__cte__pos_dedup_cdcx_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - cast(name as varchar) as name, - cast(_ab_cdc_lsn as - float -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float -) as _ab_cdc_deleted_at, - cast(_ab_cdc_log_pos as - float -) as _ab_cdc_log_pos, - _airbyte_emitted_at -from __dbt__cte__pos_dedup_cdcx_ab1 --- pos_dedup_cdcx -), __dbt__cte__pos_dedup_cdcx_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(name as varchar), '') || '-' || coalesce(cast(_ab_cdc_lsn as varchar), '') || '-' || coalesce(cast(_ab_cdc_updated_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_log_pos as varchar), '') as varchar)) as _airbyte_pos_dedup_cdcx_hashid, - tmp.* -from __dbt__cte__pos_dedup_cdcx_ab2 tmp --- pos_dedup_cdcx -), __dbt__cte__pos_dedup_cdcx_ab4 as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_pos_dedup_cdcx_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from __dbt__cte__pos_dedup_cdcx_ab3 tmp --- pos_dedup_cdcx from "integrationtests".test_normalization._airbyte_raw_pos_dedup_cdcx -)-- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid -from __dbt__cte__pos_dedup_cdcx_ab4 --- pos_dedup_cdcx from "integrationtests".test_normalization._airbyte_raw_pos_dedup_cdcx -where _airbyte_row_num = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..354cbd13be02 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,77 @@ + + + + create table + "integrationtests"."test_normalization"."dedup_cdc_excluded_scd__dbt_tmp" + + + compound sortkey(_airbyte_active_row,_airbyte_unique_key,_airbyte_emitted_at) + as ( + +with + +input_data as ( + select * + from "integrationtests"._airbyte_test_normalization."dedup_cdc_excluded_ab3" + -- dedup_cdc_excluded from "integrationtests".test_normalization._airbyte_raw_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast(id as varchar), '') as varchar)) as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as varchar), cast(_ab_cdc_updated_at as varchar) + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(coalesce(cast(_airbyte_unique_key as varchar), '') || '-' || coalesce(cast(_airbyte_start_at as varchar), '') || '-' || coalesce(cast(_airbyte_emitted_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_updated_at as varchar), '') as varchar)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..c9ec8d0622ff --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,83 @@ + + + + create table + "integrationtests"."test_normalization"."dedup_exchange_rate_scd__dbt_tmp" + + + compound sortkey(_airbyte_active_row,_airbyte_unique_key,_airbyte_emitted_at) + as ( + +with + +input_data as ( + select * + from "integrationtests"._airbyte_test_normalization."dedup_exchange_rate_ab3" + -- dedup_exchange_rate from "integrationtests".test_normalization._airbyte_raw_dedup_exchange_rate +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(currency as varchar), '') || '-' || coalesce(cast(nzd as varchar), '') as varchar)) as _airbyte_unique_key, + id, + currency, + date, + timestamp_col, + "hkd@spéçiäl & characters", + hkd_special___characters, + nzd, + usd, + date as _airbyte_start_at, + lag(date) over ( + partition by id, currency, cast(nzd as varchar) + order by + date is null asc, + date desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(date) over ( + partition by id, currency, cast(nzd as varchar) + order by + date is null asc, + date desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_exchange_rate_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(coalesce(cast(_airbyte_unique_key as varchar), '') || '-' || coalesce(cast(_airbyte_start_at as varchar), '') || '-' || coalesce(cast(_airbyte_emitted_at as varchar), '') as varchar)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + currency, + date, + timestamp_col, + "hkd@spéçiäl & characters", + hkd_special___characters, + nzd, + usd, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from dedup_data where _airbyte_row_num = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..c63c52ddd0e7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,69 @@ + + + + create table + "integrationtests"."test_normalization"."renamed_dedup_cdc_excluded_scd__dbt_tmp" + + + compound sortkey(_airbyte_active_row,_airbyte_unique_key,_airbyte_emitted_at) + as ( + +with + +input_data as ( + select * + from "integrationtests"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3" + -- renamed_dedup_cdc_excluded from "integrationtests".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast(id as varchar), '') as varchar)) as _airbyte_unique_key, + id, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_renamed_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(coalesce(cast(_airbyte_unique_key as varchar), '') || '-' || coalesce(cast(_airbyte_start_at as varchar), '') || '-' || coalesce(cast(_airbyte_emitted_at as varchar), '') as varchar)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql new file mode 100644 index 000000000000..688fbdcbe378 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -0,0 +1,29 @@ + + + + create table + "integrationtests"."test_normalization"."dedup_cdc_excluded__dbt_tmp" + + + compound sortkey(_airbyte_unique_key,_airbyte_emitted_at) + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from "integrationtests".test_normalization."dedup_cdc_excluded_scd" +-- dedup_cdc_excluded from "integrationtests".test_normalization._airbyte_raw_dedup_cdc_excluded +where 1 = 1 +and _airbyte_active_row = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..21a16d10606d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,32 @@ + + + + create table + "integrationtests"."test_normalization"."dedup_exchange_rate__dbt_tmp" + + + compound sortkey(_airbyte_unique_key,_airbyte_emitted_at) + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + currency, + date, + timestamp_col, + "hkd@spéçiäl & characters", + hkd_special___characters, + nzd, + usd, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from "integrationtests".test_normalization."dedup_exchange_rate_scd" +-- dedup_exchange_rate from "integrationtests".test_normalization._airbyte_raw_dedup_exchange_rate +where 1 = 1 +and _airbyte_active_row = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..a560a42d546a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -0,0 +1,90 @@ + + + + create table + "integrationtests"."test_normalization"."exchange_rate__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'currency', true) != '' then json_extract_path_text(_airbyte_data, 'currency', true) end as currency, + case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, + case when json_extract_path_text(_airbyte_data, 'timestamp_col', true) != '' then json_extract_path_text(_airbyte_data, 'timestamp_col', true) end as timestamp_col, + case when json_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters', true) != '' then json_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters', true) end as "hkd@spéçiäl & characters", + case when json_extract_path_text(_airbyte_data, 'HKD_special___characters', true) != '' then json_extract_path_text(_airbyte_data, 'HKD_special___characters', true) end as hkd_special___characters, + case when json_extract_path_text(_airbyte_data, 'NZD', true) != '' then json_extract_path_text(_airbyte_data, 'NZD', true) end as nzd, + case when json_extract_path_text(_airbyte_data, 'USD', true) != '' then json_extract_path_text(_airbyte_data, 'USD', true) end as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 + +), __dbt__cte__exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(currency as varchar) as currency, + cast(nullif(date, '') as + date +) as date, + cast(nullif(timestamp_col, '') as + timestamp with time zone +) as timestamp_col, + cast("hkd@spéçiäl & characters" as + float +) as "hkd@spéçiäl & characters", + cast(hkd_special___characters as varchar) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__exchange_rate_ab1 +-- exchange_rate +where 1 = 1 + +), __dbt__cte__exchange_rate_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(currency as varchar), '') || '-' || coalesce(cast(date as varchar), '') || '-' || coalesce(cast(timestamp_col as varchar), '') || '-' || coalesce(cast("hkd@spéçiäl & characters" as varchar), '') || '-' || coalesce(cast(hkd_special___characters as varchar), '') || '-' || coalesce(cast(nzd as varchar), '') || '-' || coalesce(cast(usd as varchar), '') as varchar)) as _airbyte_exchange_rate_hashid, + tmp.* +from __dbt__cte__exchange_rate_ab2 tmp +-- exchange_rate +where 1 = 1 + +)-- Final base SQL model +select + id, + currency, + date, + timestamp_col, + "hkd@spéçiäl & characters", + hkd_special___characters, + nzd, + usd, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from __dbt__cte__exchange_rate_ab3 +-- exchange_rate from "integrationtests".test_normalization._airbyte_raw_exchange_rate +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..035acaa17f2a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,25 @@ + + + + create table + "integrationtests"."test_normalization"."renamed_dedup_cdc_excluded__dbt_tmp" + + + compound sortkey(_airbyte_unique_key,_airbyte_emitted_at) + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from "integrationtests".test_normalization."renamed_dedup_cdc_excluded_scd" +-- renamed_dedup_cdc_excluded from "integrationtests".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded +where 1 = 1 +and _airbyte_active_row = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..a9657af6c9d3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,77 @@ + + + create table + "integrationtests".test_normalization."pos_dedup_cdcx_scd__dbt_tmp" + + + compound sortkey(_airbyte_active_row,_airbyte_unique_key,_airbyte_emitted_at) + as ( + +with + +input_data as ( + select * + from "integrationtests"._airbyte_test_normalization."pos_dedup_cdcx_ab3" + -- pos_dedup_cdcx from "integrationtests".test_normalization._airbyte_raw_pos_dedup_cdcx +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast(id as varchar), '') as varchar)) as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as varchar), cast(_ab_cdc_updated_at as varchar), cast(_ab_cdc_log_pos as varchar) + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(coalesce(cast(_airbyte_unique_key as varchar), '') || '-' || coalesce(cast(_airbyte_start_at as varchar), '') || '-' || coalesce(cast(_airbyte_emitted_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_updated_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_log_pos as varchar), '') as varchar)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql new file mode 100644 index 000000000000..6bc840ab2ff4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -0,0 +1,27 @@ + + + create table + "integrationtests".test_normalization."pos_dedup_cdcx__dbt_tmp" + + + compound sortkey(_airbyte_unique_key,_airbyte_emitted_at) + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from "integrationtests".test_normalization."pos_dedup_cdcx_scd" +-- pos_dedup_cdcx from "integrationtests".test_normalization._airbyte_raw_pos_dedup_cdcx +where 1 = 1 +and _airbyte_active_row = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..aef3d5e083d6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,53 @@ + + + create view "integrationtests"._airbyte_test_normalization."dedup_cdc_excluded_ab3__dbt_tmp" as ( + +with __dbt__cte__dedup_cdc_excluded_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'name', true) != '' then json_extract_path_text(_airbyte_data, 'name', true) end as name, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) end as _ab_cdc_lsn, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) end as _ab_cdc_updated_at, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) end as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias +-- dedup_cdc_excluded +where 1 = 1 + +), __dbt__cte__dedup_cdc_excluded_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(name as varchar) as name, + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__dedup_cdc_excluded_ab1 +-- dedup_cdc_excluded +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(name as varchar), '') || '-' || coalesce(cast(_ab_cdc_lsn as varchar), '') || '-' || coalesce(cast(_ab_cdc_updated_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as varchar), '') as varchar)) as _airbyte_dedup_cdc_excluded_hashid, + tmp.* +from __dbt__cte__dedup_cdc_excluded_ab2 tmp +-- dedup_cdc_excluded +where 1 = 1 + + ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..f543d9f4bbb6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,63 @@ + + + create view "integrationtests"._airbyte_test_normalization."dedup_exchange_rate_ab3__dbt_tmp" as ( + +with __dbt__cte__dedup_exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'currency', true) != '' then json_extract_path_text(_airbyte_data, 'currency', true) end as currency, + case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, + case when json_extract_path_text(_airbyte_data, 'timestamp_col', true) != '' then json_extract_path_text(_airbyte_data, 'timestamp_col', true) end as timestamp_col, + case when json_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters', true) != '' then json_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters', true) end as "hkd@spéçiäl & characters", + case when json_extract_path_text(_airbyte_data, 'HKD_special___characters', true) != '' then json_extract_path_text(_airbyte_data, 'HKD_special___characters', true) end as hkd_special___characters, + case when json_extract_path_text(_airbyte_data, 'NZD', true) != '' then json_extract_path_text(_airbyte_data, 'NZD', true) end as nzd, + case when json_extract_path_text(_airbyte_data, 'USD', true) != '' then json_extract_path_text(_airbyte_data, 'USD', true) end as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 + +), __dbt__cte__dedup_exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(currency as varchar) as currency, + cast(nullif(date, '') as + date +) as date, + cast(nullif(timestamp_col, '') as + timestamp with time zone +) as timestamp_col, + cast("hkd@spéçiäl & characters" as + float +) as "hkd@spéçiäl & characters", + cast(hkd_special___characters as varchar) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__dedup_exchange_rate_ab1 +-- dedup_exchange_rate +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(currency as varchar), '') || '-' || coalesce(cast(date as varchar), '') || '-' || coalesce(cast(timestamp_col as varchar), '') || '-' || coalesce(cast("hkd@spéçiäl & characters" as varchar), '') || '-' || coalesce(cast(hkd_special___characters as varchar), '') || '-' || coalesce(cast(nzd as varchar), '') || '-' || coalesce(cast(usd as varchar), '') as varchar)) as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from __dbt__cte__dedup_exchange_rate_ab2 tmp +-- dedup_exchange_rate +where 1 = 1 + + ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql new file mode 100644 index 000000000000..3502913a583f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -0,0 +1,54 @@ + + + create view "integrationtests"._airbyte_test_normalization."pos_dedup_cdcx_ab3__dbt_tmp" as ( + +with __dbt__cte__pos_dedup_cdcx_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'name', true) != '' then json_extract_path_text(_airbyte_data, 'name', true) end as name, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) end as _ab_cdc_lsn, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) end as _ab_cdc_updated_at, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) end as _ab_cdc_deleted_at, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_log_pos', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_log_pos', true) end as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias +-- pos_dedup_cdcx +where 1 = 1 +), __dbt__cte__pos_dedup_cdcx_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(name as varchar) as name, + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + cast(_ab_cdc_log_pos as + float +) as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__pos_dedup_cdcx_ab1 +-- pos_dedup_cdcx +where 1 = 1 +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(name as varchar), '') || '-' || coalesce(cast(_ab_cdc_lsn as varchar), '') || '-' || coalesce(cast(_ab_cdc_updated_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_log_pos as varchar), '') as varchar)) as _airbyte_pos_dedup_cdcx_hashid, + tmp.* +from __dbt__cte__pos_dedup_cdcx_ab2 tmp +-- pos_dedup_cdcx +where 1 = 1 + ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..e7841d06e0fe --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,39 @@ + + + create view "integrationtests"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3__dbt_tmp" as ( + +with __dbt__cte__renamed_dedup_cdc_excluded_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 + +), __dbt__cte__renamed_dedup_cdc_excluded_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__renamed_dedup_cdc_excluded_ab1 +-- renamed_dedup_cdc_excluded +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') as varchar)) as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from __dbt__cte__renamed_dedup_cdc_excluded_ab2 tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql deleted file mode 100644 index 6144a13617e8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql +++ /dev/null @@ -1,43 +0,0 @@ -with table_row_counts as ( - select distinct count(*) as row_count, 10 as expected_count - from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} -union all - select distinct count(*) as row_count, 10 as expected_count - from {{ ref('exchange_rate') }} - -union all - - select distinct count(*) as row_count, 10 as expected_count - from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -union all - select distinct count(*) as row_count, 10 as expected_count - from {{ ref('dedup_exchange_rate_scd') }} -union all - select distinct count(*) as row_count, 5 as expected_count - from {{ ref('dedup_exchange_rate') }} - -union all - - select distinct count(*) as row_count, 8 as expected_count - from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -union all - select distinct count(*) as row_count, 8 as expected_count - from {{ ref('dedup_cdc_excluded_scd') }} -union all - select distinct count(*) as row_count, 4 as expected_count - from {{ ref('dedup_cdc_excluded') }} - -union all - - select distinct count(*) as row_count, 8 as expected_count - from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -union all - select distinct count(*) as row_count, 8 as expected_count - from {{ ref('pos_dedup_cdcx_scd') }} -union all - select distinct count(*) as row_count, 3 as expected_count - from {{ ref('pos_dedup_cdcx') }} -) -select * -from table_row_counts -where row_count != expected_count diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql index 71f6448f6532..98cd148ff512 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -1,13 +1,21 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['name'], ['name']) }} as name, - {{ json_extract_scalar('_airbyte_data', ['column`_\'with"_quotes'], ['column___with__quotes']) }} as {{ adapter.quote('column`_\'with""_quotes') }}, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_lsn'], ['_ab_cdc_lsn']) }} as _ab_cdc_lsn, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} as table_alias -- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql index 2f1c85715033..b808389ede5a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -1,13 +1,21 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, cast(name as {{ dbt_utils.type_string() }}) as name, - cast({{ adapter.quote('column`_\'with""_quotes') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('column`_\'with""_quotes') }}, cast(_ab_cdc_lsn as {{ dbt_utils.type_float() }}) as _ab_cdc_lsn, cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('dedup_cdc_excluded_ab1') }} -- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 4e6d97a65757..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'name', - adapter.quote('column`_\'with""_quotes'), - '_ab_cdc_lsn', - '_ab_cdc_updated_at', - '_ab_cdc_deleted_at', - ]) }} as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from {{ ref('dedup_cdc_excluded_ab2') }} tmp --- dedup_cdc_excluded - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql deleted file mode 100644 index aa34179c3c5c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_cdc_excluded_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('dedup_cdc_excluded_ab3') }} tmp --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql index 5dcea3571341..463aebe85f6b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, @@ -9,7 +14,11 @@ select {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as hkd_special___characters, {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as nzd, {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} as table_alias -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index 9e15010eb3b5..35914dac7dcc 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -9,7 +14,11 @@ select cast(hkd_special___characters as {{ dbt_utils.type_string() }}) as hkd_special___characters, cast(nzd as {{ dbt_utils.type_float() }}) as nzd, cast(usd as {{ dbt_utils.type_float() }}) as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('dedup_exchange_rate_ab1') }} -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql deleted file mode 100644 index 599a1ccec515..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'currency', - 'date', - 'timestamp_col', - adapter.quote('hkd@spéçiäl & characters'), - 'hkd_special___characters', - 'nzd', - 'usd', - ]) }} as _airbyte_dedup_exchange_rate_hashid, - tmp.* -from {{ ref('dedup_exchange_rate_ab2') }} tmp --- dedup_exchange_rate - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql deleted file mode 100644 index ef5f6e52d60e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_exchange_rate_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('dedup_exchange_rate_ab3') }} tmp --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql index 70419151a6dc..354a14427e50 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, @@ -9,7 +14,11 @@ select {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as hkd_special___characters, {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as nzd, {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} as table_alias -- exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql index 907c04af5eb4..8096d935aeaa 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -9,7 +14,11 @@ select cast(hkd_special___characters as {{ dbt_utils.type_string() }}) as hkd_special___characters, cast(nzd as {{ dbt_utils.type_float() }}) as nzd, cast(usd as {{ dbt_utils.type_float() }}) as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('exchange_rate_ab1') }} -- exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql index d1b84842e82f..a07ee57a28a3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -14,4 +19,6 @@ select tmp.* from {{ ref('exchange_rate_ab2') }} tmp -- exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql index 1041493c6c67..9d9592bf5bcc 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, @@ -7,7 +12,10 @@ select {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_log_pos'], ['_ab_cdc_log_pos']) }} as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} as table_alias -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql index bccb189a9d7a..32dea2cfa7c8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -7,7 +12,10 @@ select cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, cast(_ab_cdc_log_pos as {{ dbt_utils.type_float() }}) as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('pos_dedup_cdcx_ab1') }} -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index 0fd3196cea9f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'name', - '_ab_cdc_lsn', - '_ab_cdc_updated_at', - '_ab_cdc_deleted_at', - '_ab_cdc_log_pos', - ]) }} as _airbyte_pos_dedup_cdcx_hashid, - tmp.* -from {{ ref('pos_dedup_cdcx_ab2') }} tmp --- pos_dedup_cdcx - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql deleted file mode 100644 index 7c897124b395..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_pos_dedup_cdcx_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('pos_dedup_cdcx_ab3') }} tmp --- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..c324b5203905 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,17 @@ +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..df5c1035dd92 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,17 @@ +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as {{ dbt_utils.type_bigint() }}) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ ref('renamed_dedup_cdc_excluded_ab1') }} +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..19d4d181c449 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,112 @@ +{{ config( + sort = ["_airbyte_active_row", "_airbyte_unique_key", "_airbyte_emitted_at"], + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('dedup_cdc_excluded_ab3') }} + -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('dedup_cdc_excluded_ab3') }} + -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}) + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..12f992cd0bbd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,122 @@ +{{ config( + sort = ["_airbyte_active_row", "_airbyte_unique_key", "_airbyte_emitted_at"], + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('dedup_exchange_rate_ab3') }} + -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + 'currency', + 'nzd', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('dedup_exchange_rate_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('dedup_exchange_rate_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('dedup_exchange_rate_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('dedup_exchange_rate_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('dedup_exchange_rate_ab3') }} + -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + 'currency', + 'nzd', + ]) }} as _airbyte_unique_key, + id, + currency, + date, + timestamp_col, + {{ adapter.quote('hkd@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + date as _airbyte_start_at, + lag(date) over ( + partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) + order by + date is null asc, + date desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(date) over ( + partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) + order by + date is null asc, + date desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_exchange_rate_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + currency, + date, + timestamp_col, + {{ adapter.quote('hkd@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..500c257b4a70 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,104 @@ +{{ config( + sort = ["_airbyte_active_row", "_airbyte_unique_key", "_airbyte_emitted_at"], + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('renamed_dedup_cdc_excluded_ab3') }} + -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('renamed_dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('renamed_dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('renamed_dedup_cdc_excluded_ab3') }} + -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_renamed_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql new file mode 100644 index 000000000000..385a2a1c6b24 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -0,0 +1,24 @@ +{{ config( + sort = ["_airbyte_unique_key", "_airbyte_emitted_at"], + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from {{ ref('dedup_cdc_excluded_scd') }} +-- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..64467e8148ab --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,27 @@ +{{ config( + sort = ["_airbyte_unique_key", "_airbyte_emitted_at"], + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + id, + currency, + date, + timestamp_col, + {{ adapter.quote('hkd@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from {{ ref('dedup_exchange_rate_scd') }} +-- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..62dbd07d0962 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql @@ -0,0 +1,25 @@ +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + id, + currency, + date, + timestamp_col, + {{ adapter.quote('hkd@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from {{ ref('exchange_rate_ab3') }} +-- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..12a26f6e4326 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,20 @@ +{{ config( + sort = ["_airbyte_unique_key", "_airbyte_emitted_at"], + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from {{ ref('renamed_dedup_cdc_excluded_scd') }} +-- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..5140caa6d9f4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,114 @@ +{{ config( + sort = ["_airbyte_active_row", "_airbyte_unique_key", "_airbyte_emitted_at"], + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('pos_dedup_cdcx_ab3') }} + -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('pos_dedup_cdcx_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('pos_dedup_cdcx_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('pos_dedup_cdcx_ab3') }} + -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_log_pos as {{ dbt_utils.type_string() }}) + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at', '_ab_cdc_log_pos' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index 2e53ecaf2bee..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - name, - {{ adapter.quote('column`_\'with""_quotes') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from {{ ref('dedup_cdc_excluded_scd') }} --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -where _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index 71c78bd0cb26..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - {{ adapter.quote('column`_\'with""_quotes') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from {{ ref('dedup_cdc_excluded_ab4') }} --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql deleted file mode 100644 index 0e3b855088c3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - currency, - date, - timestamp_col, - {{ adapter.quote('hkd@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from {{ ref('dedup_exchange_rate_scd') }} --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -where _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql deleted file mode 100644 index 63fab0795ade..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql +++ /dev/null @@ -1,26 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - currency, - date, - timestamp_col, - {{ adapter.quote('hkd@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, - date as _airbyte_start_at, - lag(date) over ( - partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) - order by date is null asc, date desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(date) over ( - partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) - order by date is null asc, date desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from {{ ref('dedup_exchange_rate_ab4') }} --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql deleted file mode 100644 index ffe5cafe5e4b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - currency, - date, - timestamp_col, - {{ adapter.quote('hkd@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, - _airbyte_emitted_at, - _airbyte_exchange_rate_hashid -from {{ ref('exchange_rate_ab3') }} --- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql index ed02a5d04940..1786654bece8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -1,15 +1,24 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + sort = ["_airbyte_unique_key", "_airbyte_emitted_at"], + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select + _airbyte_unique_key, id, name, _ab_cdc_lsn, _ab_cdc_updated_at, _ab_cdc_deleted_at, _ab_cdc_log_pos, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_pos_dedup_cdcx_hashid from {{ ref('pos_dedup_cdcx_scd') }} -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index 81216ec938ee..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid -from {{ ref('pos_dedup_cdcx_ab4') }} --- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..1e63c2ad6c92 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,21 @@ +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + 'name', + '_ab_cdc_lsn', + '_ab_cdc_updated_at', + '_ab_cdc_deleted_at', + ]) }} as _airbyte_dedup_cdc_excluded_hashid, + tmp.* +from {{ ref('dedup_cdc_excluded_ab2') }} tmp +-- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..e92210d17f06 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,24 @@ +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + 'currency', + 'date', + 'timestamp_col', + adapter.quote('hkd@spéçiäl & characters'), + 'hkd_special___characters', + 'nzd', + 'usd', + ]) }} as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from {{ ref('dedup_exchange_rate_ab2') }} tmp +-- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql new file mode 100644 index 000000000000..e82906073868 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -0,0 +1,21 @@ +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + 'name', + '_ab_cdc_lsn', + '_ab_cdc_updated_at', + '_ab_cdc_deleted_at', + '_ab_cdc_log_pos', + ]) }} as _airbyte_pos_dedup_cdcx_hashid, + tmp.* +from {{ ref('pos_dedup_cdcx_ab2') }} tmp +-- pos_dedup_cdcx +where 1 = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..b59cdd1d2b2c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,17 @@ +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from {{ ref('renamed_dedup_cdc_excluded_ab2') }} tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/sources.yml index c085ca8ccc25..45c338b893ca 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/sources.yml @@ -10,3 +10,4 @@ sources: - name: _airbyte_raw_dedup_exchange_rate - name: _airbyte_raw_exchange_rate - name: _airbyte_raw_pos_dedup_cdcx + - name: _airbyte_raw_renamed_dedup_cdc_excluded diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..b9485899f9c3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."dedup_cdc_excluded_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "dedup_cdc_excluded_scd__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."dedup_cdc_excluded_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid" + from "dedup_cdc_excluded_scd__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..53aa2cf9aca6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."dedup_exchange_rate_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "dedup_exchange_rate_scd__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."dedup_exchange_rate_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid" + from "dedup_exchange_rate_scd__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..70ebdfca1c57 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."renamed_dedup_cdc_excluded_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "renamed_dedup_cdc_excluded_scd__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."renamed_dedup_cdc_excluded_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid" + from "renamed_dedup_cdc_excluded_scd__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql new file mode 100644 index 000000000000..8f1ca62ed990 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."dedup_cdc_excluded" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "dedup_cdc_excluded__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."dedup_cdc_excluded" ("_airbyte_unique_key", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid") + ( + select "_airbyte_unique_key", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid" + from "dedup_cdc_excluded__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..977b187c317b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."dedup_exchange_rate" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "dedup_exchange_rate__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."dedup_exchange_rate" ("_airbyte_unique_key", "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") + ( + select "_airbyte_unique_key", "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid" + from "dedup_exchange_rate__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..0da80f1f8245 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."exchange_rate" + where (_airbyte_ab_id) in ( + select (_airbyte_ab_id) + from "exchange_rate__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."exchange_rate" ("id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid") + ( + select "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid" + from "exchange_rate__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..94ffdc307498 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."renamed_dedup_cdc_excluded" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "renamed_dedup_cdc_excluded__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."renamed_dedup_cdc_excluded" ("_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid") + ( + select "_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid" + from "renamed_dedup_cdc_excluded__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..a9657af6c9d3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,77 @@ + + + create table + "integrationtests".test_normalization."pos_dedup_cdcx_scd__dbt_tmp" + + + compound sortkey(_airbyte_active_row,_airbyte_unique_key,_airbyte_emitted_at) + as ( + +with + +input_data as ( + select * + from "integrationtests"._airbyte_test_normalization."pos_dedup_cdcx_ab3" + -- pos_dedup_cdcx from "integrationtests".test_normalization._airbyte_raw_pos_dedup_cdcx +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast(id as varchar), '') as varchar)) as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as varchar), cast(_ab_cdc_updated_at as varchar), cast(_ab_cdc_log_pos as varchar) + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(coalesce(cast(_airbyte_unique_key as varchar), '') || '-' || coalesce(cast(_airbyte_start_at as varchar), '') || '-' || coalesce(cast(_airbyte_emitted_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_updated_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_log_pos as varchar), '') as varchar)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql new file mode 100644 index 000000000000..6bc840ab2ff4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -0,0 +1,27 @@ + + + create table + "integrationtests".test_normalization."pos_dedup_cdcx__dbt_tmp" + + + compound sortkey(_airbyte_unique_key,_airbyte_emitted_at) + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from "integrationtests".test_normalization."pos_dedup_cdcx_scd" +-- pos_dedup_cdcx from "integrationtests".test_normalization._airbyte_raw_pos_dedup_cdcx +where 1 = 1 +and _airbyte_active_row = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..aef3d5e083d6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,53 @@ + + + create view "integrationtests"._airbyte_test_normalization."dedup_cdc_excluded_ab3__dbt_tmp" as ( + +with __dbt__cte__dedup_cdc_excluded_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'name', true) != '' then json_extract_path_text(_airbyte_data, 'name', true) end as name, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) end as _ab_cdc_lsn, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) end as _ab_cdc_updated_at, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) end as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias +-- dedup_cdc_excluded +where 1 = 1 + +), __dbt__cte__dedup_cdc_excluded_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(name as varchar) as name, + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__dedup_cdc_excluded_ab1 +-- dedup_cdc_excluded +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(name as varchar), '') || '-' || coalesce(cast(_ab_cdc_lsn as varchar), '') || '-' || coalesce(cast(_ab_cdc_updated_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as varchar), '') as varchar)) as _airbyte_dedup_cdc_excluded_hashid, + tmp.* +from __dbt__cte__dedup_cdc_excluded_ab2 tmp +-- dedup_cdc_excluded +where 1 = 1 + + ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..f543d9f4bbb6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,63 @@ + + + create view "integrationtests"._airbyte_test_normalization."dedup_exchange_rate_ab3__dbt_tmp" as ( + +with __dbt__cte__dedup_exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'currency', true) != '' then json_extract_path_text(_airbyte_data, 'currency', true) end as currency, + case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, + case when json_extract_path_text(_airbyte_data, 'timestamp_col', true) != '' then json_extract_path_text(_airbyte_data, 'timestamp_col', true) end as timestamp_col, + case when json_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters', true) != '' then json_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters', true) end as "hkd@spéçiäl & characters", + case when json_extract_path_text(_airbyte_data, 'HKD_special___characters', true) != '' then json_extract_path_text(_airbyte_data, 'HKD_special___characters', true) end as hkd_special___characters, + case when json_extract_path_text(_airbyte_data, 'NZD', true) != '' then json_extract_path_text(_airbyte_data, 'NZD', true) end as nzd, + case when json_extract_path_text(_airbyte_data, 'USD', true) != '' then json_extract_path_text(_airbyte_data, 'USD', true) end as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 + +), __dbt__cte__dedup_exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(currency as varchar) as currency, + cast(nullif(date, '') as + date +) as date, + cast(nullif(timestamp_col, '') as + timestamp with time zone +) as timestamp_col, + cast("hkd@spéçiäl & characters" as + float +) as "hkd@spéçiäl & characters", + cast(hkd_special___characters as varchar) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__dedup_exchange_rate_ab1 +-- dedup_exchange_rate +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(currency as varchar), '') || '-' || coalesce(cast(date as varchar), '') || '-' || coalesce(cast(timestamp_col as varchar), '') || '-' || coalesce(cast("hkd@spéçiäl & characters" as varchar), '') || '-' || coalesce(cast(hkd_special___characters as varchar), '') || '-' || coalesce(cast(nzd as varchar), '') || '-' || coalesce(cast(usd as varchar), '') as varchar)) as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from __dbt__cte__dedup_exchange_rate_ab2 tmp +-- dedup_exchange_rate +where 1 = 1 + + ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql new file mode 100644 index 000000000000..3502913a583f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -0,0 +1,54 @@ + + + create view "integrationtests"._airbyte_test_normalization."pos_dedup_cdcx_ab3__dbt_tmp" as ( + +with __dbt__cte__pos_dedup_cdcx_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'name', true) != '' then json_extract_path_text(_airbyte_data, 'name', true) end as name, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) end as _ab_cdc_lsn, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) end as _ab_cdc_updated_at, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) end as _ab_cdc_deleted_at, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_log_pos', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_log_pos', true) end as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias +-- pos_dedup_cdcx +where 1 = 1 +), __dbt__cte__pos_dedup_cdcx_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(name as varchar) as name, + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + cast(_ab_cdc_log_pos as + float +) as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__pos_dedup_cdcx_ab1 +-- pos_dedup_cdcx +where 1 = 1 +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(name as varchar), '') || '-' || coalesce(cast(_ab_cdc_lsn as varchar), '') || '-' || coalesce(cast(_ab_cdc_updated_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_log_pos as varchar), '') as varchar)) as _airbyte_pos_dedup_cdcx_hashid, + tmp.* +from __dbt__cte__pos_dedup_cdcx_ab2 tmp +-- pos_dedup_cdcx +where 1 = 1 + ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..e7841d06e0fe --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,39 @@ + + + create view "integrationtests"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3__dbt_tmp" as ( + +with __dbt__cte__renamed_dedup_cdc_excluded_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 + +), __dbt__cte__renamed_dedup_cdc_excluded_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__renamed_dedup_cdc_excluded_ab1 +-- renamed_dedup_cdc_excluded +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') as varchar)) as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from __dbt__cte__renamed_dedup_cdc_excluded_ab2 tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..35192e5d2a73 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."dedup_exchange_rate_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "dedup_exchange_rate_scd__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."dedup_exchange_rate_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid", "new_column", "id") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid", "new_column", "id" + from "dedup_exchange_rate_scd__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..3fb70e64ecc7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."renamed_dedup_cdc_excluded_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "renamed_dedup_cdc_excluded_scd__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."renamed_dedup_cdc_excluded_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at" + from "renamed_dedup_cdc_excluded_scd__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..47aaec909812 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."dedup_exchange_rate" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "dedup_exchange_rate__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."dedup_exchange_rate" ("_airbyte_unique_key", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid", "new_column", "id") + ( + select "_airbyte_unique_key", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid", "new_column", "id" + from "dedup_exchange_rate__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..3b8d494b22b5 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."exchange_rate" + where (_airbyte_ab_id) in ( + select (_airbyte_ab_id) + from "exchange_rate__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."exchange_rate" ("currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid", "new_column", "id") + ( + select "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid", "new_column", "id" + from "exchange_rate__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..8d6374d4880e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."renamed_dedup_cdc_excluded" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "renamed_dedup_cdc_excluded__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."renamed_dedup_cdc_excluded" ("_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at") + ( + select "_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at" + from "renamed_dedup_cdc_excluded__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..765e200b0da1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,65 @@ + + + create view "integrationtests"._airbyte_test_normalization."dedup_exchange_rate_ab3__dbt_tmp" as ( + +with __dbt__cte__dedup_exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'currency', true) != '' then json_extract_path_text(_airbyte_data, 'currency', true) end as currency, + case when json_extract_path_text(_airbyte_data, 'new_column', true) != '' then json_extract_path_text(_airbyte_data, 'new_column', true) end as new_column, + case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, + case when json_extract_path_text(_airbyte_data, 'timestamp_col', true) != '' then json_extract_path_text(_airbyte_data, 'timestamp_col', true) end as timestamp_col, + case when json_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters', true) != '' then json_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters', true) end as "hkd@spéçiäl & characters", + case when json_extract_path_text(_airbyte_data, 'NZD', true) != '' then json_extract_path_text(_airbyte_data, 'NZD', true) end as nzd, + case when json_extract_path_text(_airbyte_data, 'USD', true) != '' then json_extract_path_text(_airbyte_data, 'USD', true) end as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 + +), __dbt__cte__dedup_exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + float +) as id, + cast(currency as varchar) as currency, + cast(new_column as + float +) as new_column, + cast(nullif(date, '') as + date +) as date, + cast(nullif(timestamp_col, '') as + timestamp with time zone +) as timestamp_col, + cast("hkd@spéçiäl & characters" as + float +) as "hkd@spéçiäl & characters", + cast(nzd as + float +) as nzd, + cast(usd as + bigint +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__dedup_exchange_rate_ab1 +-- dedup_exchange_rate +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(currency as varchar), '') || '-' || coalesce(cast(new_column as varchar), '') || '-' || coalesce(cast(date as varchar), '') || '-' || coalesce(cast(timestamp_col as varchar), '') || '-' || coalesce(cast("hkd@spéçiäl & characters" as varchar), '') || '-' || coalesce(cast(nzd as varchar), '') || '-' || coalesce(cast(usd as varchar), '') as varchar)) as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from __dbt__cte__dedup_exchange_rate_ab2 tmp +-- dedup_exchange_rate +where 1 = 1 + + ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..0c4e15f9ecee --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,53 @@ + + + create view "integrationtests"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3__dbt_tmp" as ( + +with __dbt__cte__renamed_dedup_cdc_excluded_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'name', true) != '' then json_extract_path_text(_airbyte_data, 'name', true) end as name, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) end as _ab_cdc_lsn, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) end as _ab_cdc_updated_at, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) end as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 + +), __dbt__cte__renamed_dedup_cdc_excluded_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(name as varchar) as name, + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__renamed_dedup_cdc_excluded_ab1 +-- renamed_dedup_cdc_excluded +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(name as varchar), '') || '-' || coalesce(cast(_ab_cdc_lsn as varchar), '') || '-' || coalesce(cast(_ab_cdc_updated_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as varchar), '') as varchar)) as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from __dbt__cte__renamed_dedup_cdc_excluded_ab2 tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql deleted file mode 100644 index 6bef3219266c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - get_path(parse_json(_airbyte_data), '"conflict_stream_array"') as CONFLICT_STREAM_ARRAY, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_ARRAY as table_alias --- CONFLICT_STREAM_ARRAY - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql deleted file mode 100644 index dada3b7f692c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - varchar -) as ID, - CONFLICT_STREAM_ARRAY, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB1" --- CONFLICT_STREAM_ARRAY - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql deleted file mode 100644 index 729c495f08c2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') || '-' || coalesce(cast(CONFLICT_STREAM_ARRAY as - varchar -), '') as - varchar -)) as _AIRBYTE_CONFLICT_STREAM_ARRAY_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB2" tmp --- CONFLICT_STREAM_ARRAY - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql deleted file mode 100644 index 6759cf235073..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - - get_path(parse_json(table_alias._airbyte_data), '"conflict_stream_name"') - as CONFLICT_STREAM_NAME, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_NAME as table_alias --- CONFLICT_STREAM_NAME - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql deleted file mode 100644 index 9de606829965..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql +++ /dev/null @@ -1,15 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - varchar -) as ID, - cast(CONFLICT_STREAM_NAME as - variant -) as CONFLICT_STREAM_NAME, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB1" --- CONFLICT_STREAM_NAME - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql deleted file mode 100644 index a8f985bc6467..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') || '-' || coalesce(cast(CONFLICT_STREAM_NAME as - varchar -), '') as - varchar -)) as _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB2" tmp --- CONFLICT_STREAM_NAME - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql deleted file mode 100644 index 7490c42bcba6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, - - get_path(parse_json(table_alias.CONFLICT_STREAM_NAME), '"conflict_stream_name"') - as CONFLICT_STREAM_NAME, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME" as table_alias -where CONFLICT_STREAM_NAME is not null --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql deleted file mode 100644 index 2e8959fe5a9f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, - cast(CONFLICT_STREAM_NAME as - variant -) as CONFLICT_STREAM_NAME, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1" --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql deleted file mode 100644 index 743006f52639..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_AIRBYTE_CONFLICT_STREAM_NAME_HASHID as - varchar -), '') || '-' || coalesce(cast(CONFLICT_STREAM_NAME as - varchar -), '') as - varchar -)) as _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2" tmp --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql deleted file mode 100644 index d3cc576b8e69..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql +++ /dev/null @@ -1,12 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, - to_varchar(get_path(parse_json(CONFLICT_STREAM_NAME), '"groups"')) as GROUPS, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" as table_alias -where CONFLICT_STREAM_NAME is not null --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql deleted file mode 100644 index 6518b2b8d815..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, - cast(GROUPS as - varchar -) as GROUPS, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1" --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql deleted file mode 100644 index cbdd87f1fe23..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID as - varchar -), '') || '-' || coalesce(cast(GROUPS as - varchar -), '') as - varchar -)) as _AIRBYTE_CONFLICT_STREAM_NAME_3_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2" tmp --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql deleted file mode 100644 index 73122f365f21..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - to_varchar(get_path(parse_json(_airbyte_data), '"conflict_stream_scalar"')) as CONFLICT_STREAM_SCALAR, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_SCALAR as table_alias --- CONFLICT_STREAM_SCALAR - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql deleted file mode 100644 index 53dafe61398f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql +++ /dev/null @@ -1,15 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - varchar -) as ID, - cast(CONFLICT_STREAM_SCALAR as - bigint -) as CONFLICT_STREAM_SCALAR, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB1" --- CONFLICT_STREAM_SCALAR - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql deleted file mode 100644 index e453bd385997..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') || '-' || coalesce(cast(CONFLICT_STREAM_SCALAR as - varchar -), '') as - varchar -)) as _AIRBYTE_CONFLICT_STREAM_SCALAR_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB2" tmp --- CONFLICT_STREAM_SCALAR - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql deleted file mode 100644 index 320bca5e222a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, - - get_path(parse_json(table_alias._airbyte_data), '"partition"') - as PARTITION, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES as table_alias --- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql deleted file mode 100644 index e6a15b835c01..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - varchar -) as ID, - cast(DATE as - varchar -) as DATE, - cast(PARTITION as - variant -) as PARTITION, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1" --- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql deleted file mode 100644 index 49fb90573503..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') || '-' || coalesce(cast(DATE as - varchar -), '') || '-' || coalesce(cast(PARTITION as - varchar -), '') as - varchar -)) as _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2" tmp --- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB4.sql deleted file mode 100644 index cd4e337de08a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB4.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB4" as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID - order by _AIRBYTE_EMITTED_AT asc - ) as _AIRBYTE_ROW_NUM, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3" tmp --- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql deleted file mode 100644 index d0d5c60bb47c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, - get_path(parse_json(PARTITION), '"double_array_data"') as DOUBLE_ARRAY_DATA, - get_path(parse_json(PARTITION), '"DATA"') as DATA, - get_path(parse_json(PARTITION), '"column`_''with""_quotes"') as "column`_'with""_quotes", - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES" as table_alias -where PARTITION is not null --- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql deleted file mode 100644 index b2af08cb9c9f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, - DOUBLE_ARRAY_DATA, - DATA, - "column`_'with""_quotes", - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1" --- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql deleted file mode 100644 index 1f3f31e025f5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql +++ /dev/null @@ -1,20 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID as - varchar -), '') || '-' || coalesce(cast(DOUBLE_ARRAY_DATA as - varchar -), '') || '-' || coalesce(cast(DATA as - varchar -), '') || '-' || coalesce(cast("column`_'with""_quotes" as - varchar -), '') as - varchar -)) as _AIRBYTE_PARTITION_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2" tmp --- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql deleted file mode 100644 index 73e33d8747e7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _AIRBYTE_PARTITION_HASHID, - to_varchar(get_path(parse_json("column`_'with""_quotes".value), '"currency"')) as CURRENCY, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as table_alias -cross join table(flatten("column`_'with""_quotes")) as "column`_'with""_quotes" -where "column`_'with""_quotes" is not null --- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql deleted file mode 100644 index 37164e52f9c4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_PARTITION_HASHID, - cast(CURRENCY as - varchar -) as CURRENCY, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1" --- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql deleted file mode 100644 index 9488057417b0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_AIRBYTE_PARTITION_HASHID as - varchar -), '') || '-' || coalesce(cast(CURRENCY as - varchar -), '') as - varchar -)) as _AIRBYTE_COLUMN___WITH__QUOTES_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2" tmp --- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql deleted file mode 100644 index a1eca419d8e1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _AIRBYTE_PARTITION_HASHID, - to_varchar(get_path(parse_json(DATA.value), '"currency"')) as CURRENCY, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as table_alias -cross join table(flatten(DATA)) as DATA -where DATA is not null --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql deleted file mode 100644 index d2be8a3d89a8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_PARTITION_HASHID, - cast(CURRENCY as - varchar -) as CURRENCY, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1" --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql deleted file mode 100644 index 0635b936fcf9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_AIRBYTE_PARTITION_HASHID as - varchar -), '') || '-' || coalesce(cast(CURRENCY as - varchar -), '') as - varchar -)) as _AIRBYTE_DATA_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2" tmp --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql deleted file mode 100644 index 0fcaa6ecd82f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _AIRBYTE_PARTITION_HASHID, - to_varchar(get_path(parse_json(DOUBLE_ARRAY_DATA.value), '"id"')) as ID, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as table_alias -cross join table(flatten(DOUBLE_ARRAY_DATA)) as DOUBLE_ARRAY_DATA -where DOUBLE_ARRAY_DATA is not null --- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql deleted file mode 100644 index 8f10bbffe486..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_PARTITION_HASHID, - cast(ID as - varchar -) as ID, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1" --- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql deleted file mode 100644 index 74f6b6e45993..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_AIRBYTE_PARTITION_HASHID as - varchar -), '') || '-' || coalesce(cast(ID as - varchar -), '') as - varchar -)) as _AIRBYTE_DOUBLE_ARRAY_DATA_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2" tmp --- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql deleted file mode 100644 index d93e36223827..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES as table_alias --- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql deleted file mode 100644 index 168beb12710e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql +++ /dev/null @@ -1,15 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - varchar -) as ID, - cast(DATE as - varchar -) as DATE, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1" --- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql deleted file mode 100644 index 9a93e64a0b04..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') || '-' || coalesce(cast(DATE as - varchar -), '') as - varchar -)) as _AIRBYTE_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2" tmp --- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql deleted file mode 100644 index f58be28c589f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - get_path(parse_json(_airbyte_data), '"children"') as CHILDREN, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_UNNEST_ALIAS as table_alias --- UNNEST_ALIAS - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql deleted file mode 100644 index 5e0ec3de9d76..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - bigint -) as ID, - CHILDREN, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB1" --- UNNEST_ALIAS - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql deleted file mode 100644 index bfce4d9064ee..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') || '-' || coalesce(cast(CHILDREN as - varchar -), '') as - varchar -)) as _AIRBYTE_UNNEST_ALIAS_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB2" tmp --- UNNEST_ALIAS - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql deleted file mode 100644 index a6aa86bdf43b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _AIRBYTE_UNNEST_ALIAS_HASHID, - to_varchar(get_path(parse_json(CHILDREN.value), '"ab_id"')) as AB_ID, - - get_path(parse_json(CHILDREN.value), '"owner"') - as OWNER, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS" as table_alias -cross join table(flatten(CHILDREN)) as CHILDREN -where CHILDREN is not null --- CHILDREN at unnest_alias/children - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql deleted file mode 100644 index 65d17c2515a5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_UNNEST_ALIAS_HASHID, - cast(AB_ID as - bigint -) as AB_ID, - cast(OWNER as - variant -) as OWNER, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB1" --- CHILDREN at unnest_alias/children - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql deleted file mode 100644 index 9eaa2bdd13e8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_AIRBYTE_UNNEST_ALIAS_HASHID as - varchar -), '') || '-' || coalesce(cast(AB_ID as - varchar -), '') || '-' || coalesce(cast(OWNER as - varchar -), '') as - varchar -)) as _AIRBYTE_CHILDREN_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB2" tmp --- CHILDREN at unnest_alias/children - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql deleted file mode 100644 index 6e7d382b882c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql +++ /dev/null @@ -1,12 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _AIRBYTE_CHILDREN_HASHID, - to_varchar(get_path(parse_json(OWNER), '"owner_id"')) as OWNER_ID, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN" as table_alias -where OWNER is not null --- OWNER at unnest_alias/children/owner - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql deleted file mode 100644 index 56d5a85204cd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_CHILDREN_HASHID, - cast(OWNER_ID as - bigint -) as OWNER_ID, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB1" --- OWNER at unnest_alias/children/owner - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql deleted file mode 100644 index a1a831a0f496..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_AIRBYTE_CHILDREN_HASHID as - varchar -), '') || '-' || coalesce(cast(OWNER_ID as - varchar -), '') as - varchar -)) as _AIRBYTE_OWNER_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB2" tmp --- OWNER at unnest_alias/children/owner - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql deleted file mode 100644 index 863bca6116b1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION_NAMESPACE._AIRBYTE_RAW_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES as table_alias --- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql deleted file mode 100644 index b9cc58f6026f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql +++ /dev/null @@ -1,15 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - varchar -) as ID, - cast(DATE as - varchar -) as DATE, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1" --- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql deleted file mode 100644 index 33af774280d4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') || '-' || coalesce(cast(DATE as - varchar -), '') as - varchar -)) as _AIRBYTE_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2" tmp --- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql deleted file mode 100644 index 21233bc9ddb8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql +++ /dev/null @@ -1,14 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY" as - ( --- Final base SQL model -select - ID, - CONFLICT_STREAM_ARRAY, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_CONFLICT_STREAM_ARRAY_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB3" --- CONFLICT_STREAM_ARRAY from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_ARRAY - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql deleted file mode 100644 index 5abb9b5bebaf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql +++ /dev/null @@ -1,14 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME" as - ( --- Final base SQL model -select - ID, - CONFLICT_STREAM_NAME, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_CONFLICT_STREAM_NAME_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB3" --- CONFLICT_STREAM_NAME from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_NAME - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql deleted file mode 100644 index 6e44eb56fa7c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql +++ /dev/null @@ -1,14 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" as - ( --- Final base SQL model -select - _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, - CONFLICT_STREAM_NAME, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3" --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name from "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME" - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql deleted file mode 100644 index 6c264d164dbe..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql +++ /dev/null @@ -1,14 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" as - ( --- Final base SQL model -select - _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, - GROUPS, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_CONFLICT_STREAM_NAME_3_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3" --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name from "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql deleted file mode 100644 index f6978d8b376f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql +++ /dev/null @@ -1,14 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR" as - ( --- Final base SQL model -select - ID, - CONFLICT_STREAM_SCALAR, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_CONFLICT_STREAM_SCALAR_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB3" --- CONFLICT_STREAM_SCALAR from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_SCALAR - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql deleted file mode 100644 index 557ae8edf658..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql +++ /dev/null @@ -1,16 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES" as - ( --- Final base SQL model -select - ID, - DATE, - PARTITION, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD" --- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES -where _airbyte_active_row = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql deleted file mode 100644 index b3de9a34e619..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql +++ /dev/null @@ -1,16 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as - ( --- Final base SQL model -select - _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, - DOUBLE_ARRAY_DATA, - DATA, - "column`_'with""_quotes", - _AIRBYTE_EMITTED_AT, - _AIRBYTE_PARTITION_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3" --- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES" - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql deleted file mode 100644 index c64b65682078..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql +++ /dev/null @@ -1,14 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES" as - ( --- Final base SQL model -select - _AIRBYTE_PARTITION_HASHID, - CURRENCY, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_COLUMN___WITH__QUOTES_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3" --- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql deleted file mode 100644 index b88de2106399..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql +++ /dev/null @@ -1,14 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA" as - ( --- Final base SQL model -select - _AIRBYTE_PARTITION_HASHID, - CURRENCY, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_DATA_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3" --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql deleted file mode 100644 index 2d0123ef9cd4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql +++ /dev/null @@ -1,14 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA" as - ( --- Final base SQL model -select - _AIRBYTE_PARTITION_HASHID, - ID, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_DOUBLE_ARRAY_DATA_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3" --- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql deleted file mode 100644 index e1690d039b07..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql +++ /dev/null @@ -1,25 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD" as - ( --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - ID, - DATE, - PARTITION, - DATE as _AIRBYTE_START_AT, - lag(DATE) over ( - partition by ID - order by DATE is null asc, DATE desc, _AIRBYTE_EMITTED_AT desc - ) as _AIRBYTE_END_AT, - case when lag(DATE) over ( - partition by ID - order by DATE is null asc, DATE desc, _AIRBYTE_EMITTED_AT desc - ) is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB4" --- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES -where _airbyte_row_num = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql deleted file mode 100644 index 28541bcc6f05..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql +++ /dev/null @@ -1,14 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES" as - ( --- Final base SQL model -select - ID, - DATE, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3" --- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS.sql deleted file mode 100644 index 6540b0016cce..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS.sql +++ /dev/null @@ -1,14 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS" as - ( --- Final base SQL model -select - ID, - CHILDREN, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_UNNEST_ALIAS_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB3" --- UNNEST_ALIAS from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_UNNEST_ALIAS - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql deleted file mode 100644 index bf03f70901c6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN" as - ( --- Final base SQL model -select - _AIRBYTE_UNNEST_ALIAS_HASHID, - AB_ID, - OWNER, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_CHILDREN_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB3" --- CHILDREN at unnest_alias/children from "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS" - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql deleted file mode 100644 index 7edb0f77df92..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql +++ /dev/null @@ -1,14 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER" as - ( --- Final base SQL model -select - _AIRBYTE_CHILDREN_HASHID, - OWNER_ID, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_OWNER_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB3" --- OWNER at unnest_alias/children/owner from "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN" - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql deleted file mode 100644 index f913181bdf3f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql +++ /dev/null @@ -1,14 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES" as - ( --- Final base SQL model -select - ID, - DATE, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3" --- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES from "AIRBYTE_DATABASE".TEST_NORMALIZATION_NAMESPACE._AIRBYTE_RAW_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql new file mode 100644 index 000000000000..e18e6a208a2c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql @@ -0,0 +1,14 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + get_path(parse_json(_airbyte_data), '"conflict_stream_array"') as CONFLICT_STREAM_ARRAY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_ARRAY as table_alias +-- CONFLICT_STREAM_ARRAY +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql new file mode 100644 index 000000000000..97e976fd6462 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + varchar +) as ID, + CONFLICT_STREAM_ARRAY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB1" +-- CONFLICT_STREAM_ARRAY +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql new file mode 100644 index 000000000000..66a710356b32 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(CONFLICT_STREAM_ARRAY as + varchar +), '') as + varchar +)) as _AIRBYTE_CONFLICT_STREAM_ARRAY_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB2" tmp +-- CONFLICT_STREAM_ARRAY +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql new file mode 100644 index 000000000000..1a41bd6a787d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + + get_path(parse_json(table_alias._airbyte_data), '"conflict_stream_name"') + as CONFLICT_STREAM_NAME, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_NAME as table_alias +-- CONFLICT_STREAM_NAME +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql new file mode 100644 index 000000000000..bf735e634dcb --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql @@ -0,0 +1,18 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + varchar +) as ID, + cast(CONFLICT_STREAM_NAME as + variant +) as CONFLICT_STREAM_NAME, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB1" +-- CONFLICT_STREAM_NAME +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql new file mode 100644 index 000000000000..3fe0514dbf9f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(CONFLICT_STREAM_NAME as + varchar +), '') as + varchar +)) as _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB2" tmp +-- CONFLICT_STREAM_NAME +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql new file mode 100644 index 000000000000..e3ef93a18792 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, + + get_path(parse_json(table_alias.CONFLICT_STREAM_NAME), '"conflict_stream_name"') + as CONFLICT_STREAM_NAME, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME" as table_alias +-- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name +where 1 = 1 +and CONFLICT_STREAM_NAME is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql new file mode 100644 index 000000000000..e9b6b9804fd9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, + cast(CONFLICT_STREAM_NAME as + variant +) as CONFLICT_STREAM_NAME, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1" +-- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql new file mode 100644 index 000000000000..0616d66795ea --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_AIRBYTE_CONFLICT_STREAM_NAME_HASHID as + varchar +), '') || '-' || coalesce(cast(CONFLICT_STREAM_NAME as + varchar +), '') as + varchar +)) as _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2" tmp +-- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql new file mode 100644 index 000000000000..3e793970e09c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql @@ -0,0 +1,15 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, + to_varchar(get_path(parse_json(CONFLICT_STREAM_NAME), '"groups"')) as GROUPS, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" as table_alias +-- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and CONFLICT_STREAM_NAME is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql new file mode 100644 index 000000000000..a405f7059b95 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, + cast(GROUPS as + varchar +) as GROUPS, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1" +-- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql new file mode 100644 index 000000000000..26ab7e066acf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID as + varchar +), '') || '-' || coalesce(cast(GROUPS as + varchar +), '') as + varchar +)) as _AIRBYTE_CONFLICT_STREAM_NAME_3_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2" tmp +-- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql new file mode 100644 index 000000000000..d61b09ff806a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql @@ -0,0 +1,14 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"conflict_stream_scalar"')) as CONFLICT_STREAM_SCALAR, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_SCALAR as table_alias +-- CONFLICT_STREAM_SCALAR +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql new file mode 100644 index 000000000000..703883993977 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql @@ -0,0 +1,18 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + varchar +) as ID, + cast(CONFLICT_STREAM_SCALAR as + bigint +) as CONFLICT_STREAM_SCALAR, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB1" +-- CONFLICT_STREAM_SCALAR +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql new file mode 100644 index 000000000000..00e7c7886c89 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(CONFLICT_STREAM_SCALAR as + varchar +), '') as + varchar +)) as _AIRBYTE_CONFLICT_STREAM_SCALAR_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB2" tmp +-- CONFLICT_STREAM_SCALAR +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql new file mode 100644 index 000000000000..6e7e79743d05 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql @@ -0,0 +1,18 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, + + get_path(parse_json(table_alias._airbyte_data), '"partition"') + as PARTITION, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES as table_alias +-- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql new file mode 100644 index 000000000000..eb9c920cc525 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql @@ -0,0 +1,22 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + varchar +) as ID, + cast(DATE as + varchar +) as DATE, + cast(PARTITION as + variant +) as PARTITION, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1" +-- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql new file mode 100644 index 000000000000..7b11ebe9ba64 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, + get_path(parse_json(PARTITION), '"double_array_data"') as DOUBLE_ARRAY_DATA, + get_path(parse_json(PARTITION), '"DATA"') as DATA, + get_path(parse_json(PARTITION), '"column`_''with""_quotes"') as "column`_'with""_quotes", + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES" as table_alias +-- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and PARTITION is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql new file mode 100644 index 000000000000..8c9edac47baf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, + DOUBLE_ARRAY_DATA, + DATA, + "column`_'with""_quotes", + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1" +-- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql new file mode 100644 index 000000000000..d659cf82cab8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql @@ -0,0 +1,21 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID as + varchar +), '') || '-' || coalesce(cast(DOUBLE_ARRAY_DATA as + varchar +), '') || '-' || coalesce(cast(DATA as + varchar +), '') || '-' || coalesce(cast("column`_'with""_quotes" as + varchar +), '') as + varchar +)) as _AIRBYTE_PARTITION_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2" tmp +-- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql new file mode 100644 index 000000000000..17a36065ce62 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _AIRBYTE_PARTITION_HASHID, + to_varchar(get_path(parse_json("column`_'with""_quotes".value), '"currency"')) as CURRENCY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as table_alias +-- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +cross join table(flatten("column`_'with""_quotes")) as "column`_'with""_quotes" +where 1 = 1 +and "column`_'with""_quotes" is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql new file mode 100644 index 000000000000..de013bb4fb92 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _AIRBYTE_PARTITION_HASHID, + cast(CURRENCY as + varchar +) as CURRENCY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1" +-- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql new file mode 100644 index 000000000000..1996258464ce --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_AIRBYTE_PARTITION_HASHID as + varchar +), '') || '-' || coalesce(cast(CURRENCY as + varchar +), '') as + varchar +)) as _AIRBYTE_COLUMN___WITH__QUOTES_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2" tmp +-- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql new file mode 100644 index 000000000000..fbd05545c768 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _AIRBYTE_PARTITION_HASHID, + to_varchar(get_path(parse_json(DATA.value), '"currency"')) as CURRENCY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as table_alias +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +cross join table(flatten(DATA)) as DATA +where 1 = 1 +and DATA is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql new file mode 100644 index 000000000000..bdf9cd2b386c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _AIRBYTE_PARTITION_HASHID, + cast(CURRENCY as + varchar +) as CURRENCY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1" +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql new file mode 100644 index 000000000000..c4a6deea96b7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_AIRBYTE_PARTITION_HASHID as + varchar +), '') || '-' || coalesce(cast(CURRENCY as + varchar +), '') as + varchar +)) as _AIRBYTE_DATA_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2" tmp +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql new file mode 100644 index 000000000000..3013309f1f90 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _AIRBYTE_PARTITION_HASHID, + to_varchar(get_path(parse_json(DOUBLE_ARRAY_DATA.value), '"id"')) as ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as table_alias +-- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +cross join table(flatten(DOUBLE_ARRAY_DATA)) as DOUBLE_ARRAY_DATA +where 1 = 1 +and DOUBLE_ARRAY_DATA is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql new file mode 100644 index 000000000000..5c1e8b226a6a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _AIRBYTE_PARTITION_HASHID, + cast(ID as + varchar +) as ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1" +-- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql new file mode 100644 index 000000000000..662055a5ee44 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_AIRBYTE_PARTITION_HASHID as + varchar +), '') || '-' || coalesce(cast(ID as + varchar +), '') as + varchar +)) as _AIRBYTE_DOUBLE_ARRAY_DATA_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2" tmp +-- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql new file mode 100644 index 000000000000..44ffce39889f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql @@ -0,0 +1,14 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES as table_alias +-- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql new file mode 100644 index 000000000000..437075adbab4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql @@ -0,0 +1,18 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + varchar +) as ID, + cast(DATE as + varchar +) as DATE, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1" +-- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql new file mode 100644 index 000000000000..70f3a3546307 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(DATE as + varchar +), '') as + varchar +)) as _AIRBYTE_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2" tmp +-- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql new file mode 100644 index 000000000000..3c6d0a2e24d2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql @@ -0,0 +1,15 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + get_path(parse_json(_airbyte_data), '"children"') as CHILDREN, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_UNNEST_ALIAS as table_alias +-- UNNEST_ALIAS +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql new file mode 100644 index 000000000000..520bc1d1d7ec --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + bigint +) as ID, + CHILDREN, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB1" +-- UNNEST_ALIAS +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql new file mode 100644 index 000000000000..753d624ac210 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql @@ -0,0 +1,18 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(CHILDREN as + varchar +), '') as + varchar +)) as _AIRBYTE_UNNEST_ALIAS_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB2" tmp +-- UNNEST_ALIAS +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql new file mode 100644 index 000000000000..a4717707483e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql @@ -0,0 +1,20 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _AIRBYTE_UNNEST_ALIAS_HASHID, + to_varchar(get_path(parse_json(CHILDREN.value), '"ab_id"')) as AB_ID, + + get_path(parse_json(CHILDREN.value), '"owner"') + as OWNER, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS" as table_alias +-- CHILDREN at unnest_alias/children +cross join table(flatten(CHILDREN)) as CHILDREN +where 1 = 1 +and CHILDREN is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql new file mode 100644 index 000000000000..b0fd97c240a0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql @@ -0,0 +1,19 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _AIRBYTE_UNNEST_ALIAS_HASHID, + cast(AB_ID as + bigint +) as AB_ID, + cast(OWNER as + variant +) as OWNER, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB1" +-- CHILDREN at unnest_alias/children +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql new file mode 100644 index 000000000000..88b1593cc4d0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql @@ -0,0 +1,19 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_AIRBYTE_UNNEST_ALIAS_HASHID as + varchar +), '') || '-' || coalesce(cast(AB_ID as + varchar +), '') || '-' || coalesce(cast(OWNER as + varchar +), '') as + varchar +)) as _AIRBYTE_CHILDREN_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB2" tmp +-- CHILDREN at unnest_alias/children +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql new file mode 100644 index 000000000000..806d713ba2f3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql @@ -0,0 +1,15 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _AIRBYTE_CHILDREN_HASHID, + to_varchar(get_path(parse_json(OWNER), '"owner_id"')) as OWNER_ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN" as table_alias +-- OWNER at unnest_alias/children/owner +where 1 = 1 +and OWNER is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql new file mode 100644 index 000000000000..4e3f70fc2e82 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _AIRBYTE_CHILDREN_HASHID, + cast(OWNER_ID as + bigint +) as OWNER_ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB1" +-- OWNER at unnest_alias/children/owner +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql new file mode 100644 index 000000000000..79189e35ffbe --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_AIRBYTE_CHILDREN_HASHID as + varchar +), '') || '-' || coalesce(cast(OWNER_ID as + varchar +), '') as + varchar +)) as _AIRBYTE_OWNER_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB2" tmp +-- OWNER at unnest_alias/children/owner +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql new file mode 100644 index 000000000000..dc9d26c10728 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql @@ -0,0 +1,14 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION_NAMESPACE._AIRBYTE_RAW_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES as table_alias +-- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql new file mode 100644 index 000000000000..ac66f7e51acd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql @@ -0,0 +1,18 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + varchar +) as ID, + cast(DATE as + varchar +) as DATE, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1" +-- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql new file mode 100644 index 000000000000..0becb20f6e1b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(DATE as + varchar +), '') as + varchar +)) as _AIRBYTE_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2" tmp +-- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql new file mode 100644 index 000000000000..54641b78f236 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql @@ -0,0 +1,23 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_UNIQUE_KEY, + ID, + DATE, + PARTITION, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD" +-- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES +where 1 = 1 +and _AIRBYTE_ACTIVE_ROW = 1 + + ) order by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES" cluster by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/UNNEST_ALIAS.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/UNNEST_ALIAS.sql new file mode 100644 index 000000000000..49df6fc5d79b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/UNNEST_ALIAS.sql @@ -0,0 +1,20 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS" as + (select * from( + +-- Final base SQL model +select + ID, + CHILDREN, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_UNNEST_ALIAS_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB3" +-- UNNEST_ALIAS from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_UNNEST_ALIAS +where 1 = 1 + + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql new file mode 100644 index 000000000000..a3f67db69287 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql @@ -0,0 +1,81 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD" as + (select * from( + +with + +input_data as ( + select * + from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3" + -- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast(ID as + varchar +), '') as + varchar +)) as _AIRBYTE_UNIQUE_KEY, + ID, + DATE, + PARTITION, + DATE as _AIRBYTE_START_AT, + lag(DATE) over ( + partition by ID + order by + DATE is null asc, + DATE desc, + _AIRBYTE_EMITTED_AT desc + ) as _AIRBYTE_END_AT, + case when lag(DATE) over ( + partition by ID + order by + DATE is null asc, + DATE desc, + _AIRBYTE_EMITTED_AT desc + ) is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT + order by _AIRBYTE_AB_ID + ) as _AIRBYTE_ROW_NUM, + md5(cast(coalesce(cast(_AIRBYTE_UNIQUE_KEY as + varchar +), '') || '-' || coalesce(cast(_AIRBYTE_START_AT as + varchar +), '') || '-' || coalesce(cast(_AIRBYTE_EMITTED_AT as + varchar +), '') as + varchar +)) as _AIRBYTE_UNIQUE_KEY_SCD, + scd_data.* + from scd_data +) +select + _AIRBYTE_UNIQUE_KEY, + _AIRBYTE_UNIQUE_KEY_SCD, + ID, + DATE, + PARTITION, + _AIRBYTE_START_AT, + _AIRBYTE_END_AT, + _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID +from dedup_data where _AIRBYTE_ROW_NUM = 1 + ) order by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD" cluster by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql new file mode 100644 index 000000000000..a366f2bba8cc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY" as + (select * from( + +-- Final base SQL model +select + ID, + CONFLICT_STREAM_ARRAY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_CONFLICT_STREAM_ARRAY_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB3" +-- CONFLICT_STREAM_ARRAY from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_ARRAY +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql new file mode 100644 index 000000000000..a20080a7f7f4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME" as + (select * from( + +-- Final base SQL model +select + ID, + CONFLICT_STREAM_NAME, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_CONFLICT_STREAM_NAME_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB3" +-- CONFLICT_STREAM_NAME from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_NAME +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql new file mode 100644 index 000000000000..4d69fa730172 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, + CONFLICT_STREAM_NAME, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3" +-- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name from "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql new file mode 100644 index 000000000000..97d25955385f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, + GROUPS, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_CONFLICT_STREAM_NAME_3_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3" +-- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name from "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql new file mode 100644 index 000000000000..efc7cbe9dce6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR" as + (select * from( + +-- Final base SQL model +select + ID, + CONFLICT_STREAM_SCALAR, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_CONFLICT_STREAM_SCALAR_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB3" +-- CONFLICT_STREAM_SCALAR from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_SCALAR +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql new file mode 100644 index 000000000000..75228a851232 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql @@ -0,0 +1,21 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, + DOUBLE_ARRAY_DATA, + DATA, + "column`_'with""_quotes", + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_PARTITION_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3" +-- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql new file mode 100644 index 000000000000..1f2adeefe899 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_PARTITION_HASHID, + CURRENCY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_COLUMN___WITH__QUOTES_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3" +-- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql new file mode 100644 index 000000000000..cfc1ce750875 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_PARTITION_HASHID, + CURRENCY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_DATA_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3" +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql new file mode 100644 index 000000000000..4bdef077e4b3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_PARTITION_HASHID, + ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_DOUBLE_ARRAY_DATA_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3" +-- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql new file mode 100644 index 000000000000..a470f14eceac --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES" as + (select * from( + +-- Final base SQL model +select + ID, + DATE, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3" +-- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql new file mode 100644 index 000000000000..0657b14de313 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql @@ -0,0 +1,20 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_UNNEST_ALIAS_HASHID, + AB_ID, + OWNER, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_CHILDREN_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB3" +-- CHILDREN at unnest_alias/children from "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql new file mode 100644 index 000000000000..652a9bd48931 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_CHILDREN_HASHID, + OWNER_ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_OWNER_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB3" +-- OWNER at unnest_alias/children/owner from "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql new file mode 100644 index 000000000000..057fea2c9ee7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES" as + (select * from( + +-- Final base SQL model +select + ID, + DATE, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3" +-- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES from "AIRBYTE_DATABASE".TEST_NORMALIZATION_NAMESPACE._AIRBYTE_RAW_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql new file mode 100644 index 000000000000..b224c3ada40e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql @@ -0,0 +1,20 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(DATE as + varchar +), '') || '-' || coalesce(cast(PARTITION as + varchar +), '') as + varchar +)) as _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2" tmp +-- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql deleted file mode 100644 index fceb5499a1f2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql +++ /dev/null @@ -1,19 +0,0 @@ -with table_row_counts as ( - select distinct count(*) as row_count, 2 as expected_count - from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES') }} -union all - select distinct count(*) as row_count, 2 as expected_count - from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES') }} -union all - select distinct count(*) as row_count, 2 as expected_count - from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION') }} -union all - select count(distinct currency) as row_count, 1 as expected_count - from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA') }} --- union all --- select count(distinct id) as row_count, 3 as expected_count --- from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA') }} -) -select * -from table_row_counts -where row_count != expected_count diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql index 61230105c911..991a40b91868 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, {{ json_extract_array('_airbyte_data', ['conflict_stream_array'], ['conflict_stream_array']) }} as CONFLICT_STREAM_ARRAY, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_CONFLICT_STREAM_ARRAY') }} as table_alias -- CONFLICT_STREAM_ARRAY +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql index 8978f4dbac41..645bc6ce95f4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(ID as {{ dbt_utils.type_string() }}) as ID, CONFLICT_STREAM_ARRAY, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('CONFLICT_STREAM_ARRAY_AB1') }} -- CONFLICT_STREAM_ARRAY +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql index 9c9ee59dc86d..83752bf1352d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('CONFLICT_STREAM_ARRAY_AB2') }} tmp -- CONFLICT_STREAM_ARRAY +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql index 338b614c5cf9..5822167058d6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, {{ json_extract('table_alias', '_airbyte_data', ['conflict_stream_name'], ['conflict_stream_name']) }} as CONFLICT_STREAM_NAME, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_CONFLICT_STREAM_NAME') }} as table_alias -- CONFLICT_STREAM_NAME +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql index 9279f897b33d..73e500cda859 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(ID as {{ dbt_utils.type_string() }}) as ID, cast(CONFLICT_STREAM_NAME as {{ type_json() }}) as CONFLICT_STREAM_NAME, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('CONFLICT_STREAM_NAME_AB1') }} -- CONFLICT_STREAM_NAME +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql index 7ae3ea76192d..516db74e7a46 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('CONFLICT_STREAM_NAME_AB2') }} tmp -- CONFLICT_STREAM_NAME +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql index 10e877a6018e..8c5659840ae3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql @@ -1,10 +1,18 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, {{ json_extract('table_alias', 'CONFLICT_STREAM_NAME', ['conflict_stream_name'], ['conflict_stream_name']) }} as CONFLICT_STREAM_NAME, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('CONFLICT_STREAM_NAME') }} as table_alias -where CONFLICT_STREAM_NAME is not null -- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name +where 1 = 1 +and CONFLICT_STREAM_NAME is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql index 32736a9e7dab..e23598a11d34 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, cast(CONFLICT_STREAM_NAME as {{ type_json() }}) as CONFLICT_STREAM_NAME, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1') }} -- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql index fa8fd857e30d..97ca26979ab2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2') }} tmp -- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql index 094d1bf989fc..df7f67fc3449 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql @@ -1,10 +1,18 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, {{ json_extract_scalar('CONFLICT_STREAM_NAME', ['groups'], ['groups']) }} as GROUPS, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME') }} as table_alias -where CONFLICT_STREAM_NAME is not null -- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and CONFLICT_STREAM_NAME is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql index 24867ad66cb7..1956ff59d185 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, cast(GROUPS as {{ dbt_utils.type_string() }}) as GROUPS, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1') }} -- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql index 9f2fd41df405..e4a14d193cdc 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2') }} tmp -- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql index c09c78954b90..1452594dfb23 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, {{ json_extract_scalar('_airbyte_data', ['conflict_stream_scalar'], ['conflict_stream_scalar']) }} as CONFLICT_STREAM_SCALAR, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_CONFLICT_STREAM_SCALAR') }} as table_alias -- CONFLICT_STREAM_SCALAR +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql index 826e19497f0a..d889d1f350ce 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(ID as {{ dbt_utils.type_string() }}) as ID, cast(CONFLICT_STREAM_SCALAR as {{ dbt_utils.type_bigint() }}) as CONFLICT_STREAM_SCALAR, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('CONFLICT_STREAM_SCALAR_AB1') }} -- CONFLICT_STREAM_SCALAR +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql index 225410177ee0..5b6d06708f98 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('CONFLICT_STREAM_SCALAR_AB2') }} tmp -- CONFLICT_STREAM_SCALAR +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql index 2b9b9145d113..b74a533e30d4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql @@ -1,10 +1,19 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as DATE, {{ json_extract('table_alias', '_airbyte_data', ['partition'], ['partition']) }} as PARTITION, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES') }} as table_alias -- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql index b294f52bd063..6acd4a631460 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql @@ -1,10 +1,19 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(ID as {{ dbt_utils.type_string() }}) as ID, cast(DATE as {{ dbt_utils.type_string() }}) as DATE, cast(PARTITION as {{ type_json() }}) as PARTITION, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1') }} -- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql deleted file mode 100644 index 353e4f8c3117..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql +++ /dev/null @@ -1,12 +0,0 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'ID', - 'DATE', - 'PARTITION', - ]) }} as _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, - tmp.* -from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2') }} tmp --- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB4.sql deleted file mode 100644 index 2e20f30cde09..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID - order by _AIRBYTE_EMITTED_AT asc - ) as _AIRBYTE_ROW_NUM, - tmp.* -from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3') }} tmp --- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql index 18f29ba28605..d9183add3292 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql @@ -1,12 +1,20 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, {{ json_extract_array('PARTITION', ['double_array_data'], ['double_array_data']) }} as DOUBLE_ARRAY_DATA, {{ json_extract_array('PARTITION', ['DATA'], ['DATA']) }} as DATA, {{ json_extract_array('PARTITION', ['column`_\'with"_quotes'], ['column___with__quotes']) }} as {{ adapter.quote('column`_\'with""_quotes') }}, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES') }} as table_alias -where PARTITION is not null -- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and PARTITION is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql index 798cdc9e347f..f610a8eb8bca 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql @@ -1,11 +1,19 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, DOUBLE_ARRAY_DATA, DATA, {{ adapter.quote('column`_\'with""_quotes') }}, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1') }} -- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql index 68df2747dfe5..8688fa6761c6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -10,4 +15,5 @@ select tmp.* from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2') }} tmp -- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql index dfbff2614d4e..6a50aee02e79 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql @@ -1,12 +1,20 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION', 'PARTITION', adapter.quote('column`_\'with""_quotes')) }} select _AIRBYTE_PARTITION_HASHID, {{ json_extract_scalar(unnested_column_value(adapter.quote('column`_\'with""_quotes')), ['currency'], ['currency']) }} as CURRENCY, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION') }} as table_alias -{{ cross_join_unnest('PARTITION', adapter.quote('column`_\'with""_quotes')) }} -where {{ adapter.quote('column`_\'with""_quotes') }} is not null -- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +{{ cross_join_unnest('PARTITION', adapter.quote('column`_\'with""_quotes')) }} +where 1 = 1 +and {{ adapter.quote('column`_\'with""_quotes') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql index 98f23739dc48..301b85b8e983 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _AIRBYTE_PARTITION_HASHID, cast(CURRENCY as {{ dbt_utils.type_string() }}) as CURRENCY, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1') }} -- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql index 47cae36ad121..b4f7cd0bdb61 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2') }} tmp -- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql index ba79ad031f77..daa25f87285e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql @@ -1,12 +1,20 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION', 'PARTITION', 'DATA') }} select _AIRBYTE_PARTITION_HASHID, {{ json_extract_scalar(unnested_column_value('DATA'), ['currency'], ['currency']) }} as CURRENCY, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION') }} as table_alias -{{ cross_join_unnest('PARTITION', 'DATA') }} -where DATA is not null -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +{{ cross_join_unnest('PARTITION', 'DATA') }} +where 1 = 1 +and DATA is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql index cc05da50b5da..81f8c127d424 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _AIRBYTE_PARTITION_HASHID, cast(CURRENCY as {{ dbt_utils.type_string() }}) as CURRENCY, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1') }} -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql index 6513fa3099fc..11dd0dc3fc02 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2') }} tmp -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql index 6e51af8df4f2..281b8400bb15 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql @@ -1,12 +1,20 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION', 'PARTITION', 'DOUBLE_ARRAY_DATA') }} select _AIRBYTE_PARTITION_HASHID, {{ json_extract_scalar(unnested_column_value('DOUBLE_ARRAY_DATA'), ['id'], ['id']) }} as ID, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION') }} as table_alias -{{ cross_join_unnest('PARTITION', 'DOUBLE_ARRAY_DATA') }} -where DOUBLE_ARRAY_DATA is not null -- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +{{ cross_join_unnest('PARTITION', 'DOUBLE_ARRAY_DATA') }} +where 1 = 1 +and DOUBLE_ARRAY_DATA is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql index 2c37ca2f2a42..f036c8235e70 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _AIRBYTE_PARTITION_HASHID, cast(ID as {{ dbt_utils.type_string() }}) as ID, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1') }} -- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql index 7c756d2b9cc9..143f944237e4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2') }} tmp -- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql index 9820def18563..6692c6fcf348 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as DATE, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES') }} as table_alias -- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql index 1e1ebdd8fe21..b3926083d3ef 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(ID as {{ dbt_utils.type_string() }}) as ID, cast(DATE as {{ dbt_utils.type_string() }}) as DATE, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1') }} -- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql index e033edcff94c..bd557dcba2d1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2') }} tmp -- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql index 3fb9992e3ed6..f410de949a5f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql @@ -1,9 +1,18 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, {{ json_extract_array('_airbyte_data', ['children'], ['children']) }} as CHILDREN, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_UNNEST_ALIAS') }} as table_alias -- UNNEST_ALIAS +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql index 176d126943ef..15ef88171eae 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql @@ -1,9 +1,18 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(ID as {{ dbt_utils.type_bigint() }}) as ID, CHILDREN, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('UNNEST_ALIAS_AB1') }} -- UNNEST_ALIAS +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql index 89c52952e230..8b4fd76705ce 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,6 @@ select tmp.* from {{ ref('UNNEST_ALIAS_AB2') }} tmp -- UNNEST_ALIAS +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql index 140ffe367d6f..3927a7ba55db 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql @@ -1,13 +1,21 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('UNNEST_ALIAS', 'UNNEST_ALIAS', 'CHILDREN') }} select _AIRBYTE_UNNEST_ALIAS_HASHID, {{ json_extract_scalar(unnested_column_value('CHILDREN'), ['ab_id'], ['ab_id']) }} as AB_ID, {{ json_extract('', unnested_column_value('CHILDREN'), ['owner'], ['owner']) }} as OWNER, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('UNNEST_ALIAS') }} as table_alias -{{ cross_join_unnest('UNNEST_ALIAS', 'CHILDREN') }} -where CHILDREN is not null -- CHILDREN at unnest_alias/children +{{ cross_join_unnest('UNNEST_ALIAS', 'CHILDREN') }} +where 1 = 1 +and CHILDREN is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql index d3dac77c6240..f4726f1b470e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql @@ -1,10 +1,18 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _AIRBYTE_UNNEST_ALIAS_HASHID, cast(AB_ID as {{ dbt_utils.type_bigint() }}) as AB_ID, cast(OWNER as {{ type_json() }}) as OWNER, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('UNNEST_ALIAS_CHILDREN_AB1') }} -- CHILDREN at unnest_alias/children +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql index bfcc222e2e4f..57b65d2b40db 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -9,4 +14,5 @@ select tmp.* from {{ ref('UNNEST_ALIAS_CHILDREN_AB2') }} tmp -- CHILDREN at unnest_alias/children +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql index b0f2a4bee14f..a80f261e0935 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql @@ -1,10 +1,18 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _AIRBYTE_CHILDREN_HASHID, {{ json_extract_scalar('OWNER', ['owner_id'], ['owner_id']) }} as OWNER_ID, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('UNNEST_ALIAS_CHILDREN') }} as table_alias -where OWNER is not null -- OWNER at unnest_alias/children/owner +where 1 = 1 +and OWNER is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql index 930146382f99..156bdcb32bce 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _AIRBYTE_CHILDREN_HASHID, cast(OWNER_ID as {{ dbt_utils.type_bigint() }}) as OWNER_ID, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('UNNEST_ALIAS_CHILDREN_OWNER_AB1') }} -- OWNER at unnest_alias/children/owner +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql index e21dff47783c..07a3fe2b02c4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('UNNEST_ALIAS_CHILDREN_OWNER_AB2') }} tmp -- OWNER at unnest_alias/children/owner +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql index 4fddf6417250..1c660548d9ef 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION_NAMESPACE", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION_NAMESPACE", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as DATE, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ source('TEST_NORMALIZATION_NAMESPACE', '_AIRBYTE_RAW_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES') }} as table_alias -- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql index 3586cbb1a476..d4ba14c99afb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION_NAMESPACE", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION_NAMESPACE", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(ID as {{ dbt_utils.type_string() }}) as ID, cast(DATE as {{ dbt_utils.type_string() }}) as DATE, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1') }} -- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql index b75132fc0ce9..56780a4b7cf3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION_NAMESPACE", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION_NAMESPACE", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2') }} tmp -- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql new file mode 100644 index 000000000000..0908a9ca3ec4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql @@ -0,0 +1,22 @@ +{{ config( + cluster_by = ["_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_EMITTED_AT"], + unique_key = "_AIRBYTE_UNIQUE_KEY", + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _AIRBYTE_UNIQUE_KEY, + ID, + DATE, + PARTITION, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID +from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD') }} +-- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES') }} +where 1 = 1 +and _AIRBYTE_ACTIVE_ROW = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/UNNEST_ALIAS.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/UNNEST_ALIAS.sql new file mode 100644 index 000000000000..eaaa8ed6bc11 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/UNNEST_ALIAS.sql @@ -0,0 +1,19 @@ +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + ID, + CHILDREN, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_UNNEST_ALIAS_HASHID +from {{ ref('UNNEST_ALIAS_AB3') }} +-- UNNEST_ALIAS from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_UNNEST_ALIAS') }} +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql new file mode 100644 index 000000000000..a47501b4ad35 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql @@ -0,0 +1,108 @@ +{{ config( + cluster_by = ["_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_EMITTED_AT"], + unique_key = "_AIRBYTE_UNIQUE_KEY_SCD", + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3') }} + -- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES') }} + where 1 = 1 + {{ incremental_clause('_AIRBYTE_EMITTED_AT') }} +), +new_data_ids as ( + -- build a subset of _AIRBYTE_UNIQUE_KEY from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'ID', + ]) }} as _AIRBYTE_UNIQUE_KEY + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._AIRBYTE_UNIQUE_KEY = new_data_ids._AIRBYTE_UNIQUE_KEY + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3') }} as inc_data on 1 = 0 + where _AIRBYTE_ACTIVE_ROW = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3')) }} from new_data + union all + select {{ dbt_utils.star(ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3') }} + -- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'ID', + ]) }} as _AIRBYTE_UNIQUE_KEY, + ID, + DATE, + PARTITION, + DATE as _AIRBYTE_START_AT, + lag(DATE) over ( + partition by ID + order by + DATE is null asc, + DATE desc, + _AIRBYTE_EMITTED_AT desc + ) as _AIRBYTE_END_AT, + case when lag(DATE) over ( + partition by ID + order by + DATE is null asc, + DATE desc, + _AIRBYTE_EMITTED_AT desc + ) is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT + order by _AIRBYTE_AB_ID + ) as _AIRBYTE_ROW_NUM, + {{ dbt_utils.surrogate_key([ + '_AIRBYTE_UNIQUE_KEY', + '_AIRBYTE_START_AT', + '_AIRBYTE_EMITTED_AT' + ]) }} as _AIRBYTE_UNIQUE_KEY_SCD, + scd_data.* + from scd_data +) +select + _AIRBYTE_UNIQUE_KEY, + _AIRBYTE_UNIQUE_KEY_SCD, + ID, + DATE, + PARTITION, + _AIRBYTE_START_AT, + _AIRBYTE_END_AT, + _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID +from dedup_data where _AIRBYTE_ROW_NUM = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql index 258221562b2d..9f76e65e0fcd 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql @@ -1,10 +1,18 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["top-level"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} -- Final base SQL model select ID, CONFLICT_STREAM_ARRAY, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_CONFLICT_STREAM_ARRAY_HASHID from {{ ref('CONFLICT_STREAM_ARRAY_AB3') }} -- CONFLICT_STREAM_ARRAY from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_CONFLICT_STREAM_ARRAY') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql index c35f1fb8aaaa..713b00119ec0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql @@ -1,10 +1,18 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["top-level"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} -- Final base SQL model select ID, CONFLICT_STREAM_NAME, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_CONFLICT_STREAM_NAME_HASHID from {{ ref('CONFLICT_STREAM_NAME_AB3') }} -- CONFLICT_STREAM_NAME from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_CONFLICT_STREAM_NAME') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql index 5d4ac2bec638..b28901245add 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql @@ -1,10 +1,18 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["nested"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION", + tags = [ "nested" ] +) }} -- Final base SQL model select _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, CONFLICT_STREAM_NAME, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID from {{ ref('CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3') }} -- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name from {{ ref('CONFLICT_STREAM_NAME') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql index 1cbd75908309..9e873bdf985d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql @@ -1,10 +1,18 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["nested"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION", + tags = [ "nested" ] +) }} -- Final base SQL model select _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, GROUPS, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_CONFLICT_STREAM_NAME_3_HASHID from {{ ref('CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3') }} -- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name from {{ ref('CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql index 13afbef4392a..57ea8746c22d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql @@ -1,10 +1,18 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["top-level"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} -- Final base SQL model select ID, CONFLICT_STREAM_SCALAR, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_CONFLICT_STREAM_SCALAR_HASHID from {{ ref('CONFLICT_STREAM_SCALAR_AB3') }} -- CONFLICT_STREAM_SCALAR from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_CONFLICT_STREAM_SCALAR') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql deleted file mode 100644 index 6bf3a89d196f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql +++ /dev/null @@ -1,12 +0,0 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["top-level"]) }} --- Final base SQL model -select - ID, - DATE, - PARTITION, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID -from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD') }} --- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES') }} -where _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql index 456f08e6bb2d..05cd2e777323 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql @@ -1,12 +1,20 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["nested"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION", + tags = [ "nested" ] +) }} -- Final base SQL model select _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, DOUBLE_ARRAY_DATA, DATA, {{ adapter.quote('column`_\'with""_quotes') }}, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_PARTITION_HASHID from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3') }} -- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql index cb8d1021e10e..cfcb9c78f631 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql @@ -1,10 +1,18 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["nested"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION", + tags = [ "nested" ] +) }} -- Final base SQL model select _AIRBYTE_PARTITION_HASHID, CURRENCY, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_COLUMN___WITH__QUOTES_HASHID from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3') }} -- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql index 97b9fb2d61fb..fb8b719df25e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql @@ -1,10 +1,18 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["nested"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION", + tags = [ "nested" ] +) }} -- Final base SQL model select _AIRBYTE_PARTITION_HASHID, CURRENCY, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_DATA_HASHID from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3') }} -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql index ce4af8494123..4282011cdce6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql @@ -1,10 +1,18 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["nested"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION", + tags = [ "nested" ] +) }} -- Final base SQL model select _AIRBYTE_PARTITION_HASHID, ID, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_DOUBLE_ARRAY_DATA_HASHID from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3') }} -- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql deleted file mode 100644 index 5ab034b5cb5a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - ID, - DATE, - PARTITION, - DATE as _AIRBYTE_START_AT, - lag(DATE) over ( - partition by ID - order by DATE is null asc, DATE desc, _AIRBYTE_EMITTED_AT desc - ) as _AIRBYTE_END_AT, - case when lag(DATE) over ( - partition by ID - order by DATE is null asc, DATE desc, _AIRBYTE_EMITTED_AT desc - ) is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID -from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB4') }} --- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql index fdce34e55683..e8419fbe61a8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql @@ -1,10 +1,18 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["top-level"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} -- Final base SQL model select ID, DATE, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID from {{ ref('NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3') }} -- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS.sql deleted file mode 100644 index 0eca0476eb43..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS.sql +++ /dev/null @@ -1,10 +0,0 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["top-level"]) }} --- Final base SQL model -select - ID, - CHILDREN, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_UNNEST_ALIAS_HASHID -from {{ ref('UNNEST_ALIAS_AB3') }} --- UNNEST_ALIAS from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_UNNEST_ALIAS') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql index 6a76c5c0ed8a..d94783e6631d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql @@ -1,11 +1,19 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["nested"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION", + tags = [ "nested" ] +) }} -- Final base SQL model select _AIRBYTE_UNNEST_ALIAS_HASHID, AB_ID, OWNER, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_CHILDREN_HASHID from {{ ref('UNNEST_ALIAS_CHILDREN_AB3') }} -- CHILDREN at unnest_alias/children from {{ ref('UNNEST_ALIAS') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql index 26769a1920c7..639a820f74be 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql @@ -1,10 +1,18 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["nested"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION", + tags = [ "nested" ] +) }} -- Final base SQL model select _AIRBYTE_CHILDREN_HASHID, OWNER_ID, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_OWNER_HASHID from {{ ref('UNNEST_ALIAS_CHILDREN_OWNER_AB3') }} -- OWNER at unnest_alias/children/owner from {{ ref('UNNEST_ALIAS_CHILDREN') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql index ba2beaf304f5..67e8f286cd08 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql @@ -1,10 +1,18 @@ -{{ config(schema="TEST_NORMALIZATION_NAMESPACE", tags=["top-level"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION_NAMESPACE", + tags = [ "top-level" ] +) }} -- Final base SQL model select ID, DATE, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID from {{ ref('SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3') }} -- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES from {{ source('TEST_NORMALIZATION_NAMESPACE', '_AIRBYTE_RAW_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql new file mode 100644 index 000000000000..adb912d6a122 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql @@ -0,0 +1,19 @@ +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'ID', + 'DATE', + 'PARTITION', + ]) }} as _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, + tmp.* +from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2') }} tmp +-- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql new file mode 100644 index 000000000000..e18e6a208a2c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql @@ -0,0 +1,14 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + get_path(parse_json(_airbyte_data), '"conflict_stream_array"') as CONFLICT_STREAM_ARRAY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_ARRAY as table_alias +-- CONFLICT_STREAM_ARRAY +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql new file mode 100644 index 000000000000..97e976fd6462 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + varchar +) as ID, + CONFLICT_STREAM_ARRAY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB1" +-- CONFLICT_STREAM_ARRAY +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql new file mode 100644 index 000000000000..66a710356b32 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(CONFLICT_STREAM_ARRAY as + varchar +), '') as + varchar +)) as _AIRBYTE_CONFLICT_STREAM_ARRAY_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB2" tmp +-- CONFLICT_STREAM_ARRAY +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql new file mode 100644 index 000000000000..1a41bd6a787d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + + get_path(parse_json(table_alias._airbyte_data), '"conflict_stream_name"') + as CONFLICT_STREAM_NAME, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_NAME as table_alias +-- CONFLICT_STREAM_NAME +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql new file mode 100644 index 000000000000..bf735e634dcb --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql @@ -0,0 +1,18 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + varchar +) as ID, + cast(CONFLICT_STREAM_NAME as + variant +) as CONFLICT_STREAM_NAME, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB1" +-- CONFLICT_STREAM_NAME +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql new file mode 100644 index 000000000000..3fe0514dbf9f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(CONFLICT_STREAM_NAME as + varchar +), '') as + varchar +)) as _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB2" tmp +-- CONFLICT_STREAM_NAME +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql new file mode 100644 index 000000000000..e3ef93a18792 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, + + get_path(parse_json(table_alias.CONFLICT_STREAM_NAME), '"conflict_stream_name"') + as CONFLICT_STREAM_NAME, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME" as table_alias +-- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name +where 1 = 1 +and CONFLICT_STREAM_NAME is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql new file mode 100644 index 000000000000..e9b6b9804fd9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, + cast(CONFLICT_STREAM_NAME as + variant +) as CONFLICT_STREAM_NAME, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1" +-- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql new file mode 100644 index 000000000000..0616d66795ea --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_AIRBYTE_CONFLICT_STREAM_NAME_HASHID as + varchar +), '') || '-' || coalesce(cast(CONFLICT_STREAM_NAME as + varchar +), '') as + varchar +)) as _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2" tmp +-- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql new file mode 100644 index 000000000000..3e793970e09c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql @@ -0,0 +1,15 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, + to_varchar(get_path(parse_json(CONFLICT_STREAM_NAME), '"groups"')) as GROUPS, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" as table_alias +-- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and CONFLICT_STREAM_NAME is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql new file mode 100644 index 000000000000..a405f7059b95 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, + cast(GROUPS as + varchar +) as GROUPS, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1" +-- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql new file mode 100644 index 000000000000..26ab7e066acf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID as + varchar +), '') || '-' || coalesce(cast(GROUPS as + varchar +), '') as + varchar +)) as _AIRBYTE_CONFLICT_STREAM_NAME_3_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2" tmp +-- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql new file mode 100644 index 000000000000..d61b09ff806a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql @@ -0,0 +1,14 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"conflict_stream_scalar"')) as CONFLICT_STREAM_SCALAR, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_SCALAR as table_alias +-- CONFLICT_STREAM_SCALAR +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql new file mode 100644 index 000000000000..703883993977 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql @@ -0,0 +1,18 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + varchar +) as ID, + cast(CONFLICT_STREAM_SCALAR as + bigint +) as CONFLICT_STREAM_SCALAR, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB1" +-- CONFLICT_STREAM_SCALAR +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql new file mode 100644 index 000000000000..00e7c7886c89 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(CONFLICT_STREAM_SCALAR as + varchar +), '') as + varchar +)) as _AIRBYTE_CONFLICT_STREAM_SCALAR_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB2" tmp +-- CONFLICT_STREAM_SCALAR +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql new file mode 100644 index 000000000000..6e7e79743d05 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql @@ -0,0 +1,18 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, + + get_path(parse_json(table_alias._airbyte_data), '"partition"') + as PARTITION, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES as table_alias +-- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql new file mode 100644 index 000000000000..eb9c920cc525 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql @@ -0,0 +1,22 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + varchar +) as ID, + cast(DATE as + varchar +) as DATE, + cast(PARTITION as + variant +) as PARTITION, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1" +-- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql new file mode 100644 index 000000000000..7b11ebe9ba64 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, + get_path(parse_json(PARTITION), '"double_array_data"') as DOUBLE_ARRAY_DATA, + get_path(parse_json(PARTITION), '"DATA"') as DATA, + get_path(parse_json(PARTITION), '"column`_''with""_quotes"') as "column`_'with""_quotes", + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES" as table_alias +-- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and PARTITION is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql new file mode 100644 index 000000000000..8c9edac47baf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, + DOUBLE_ARRAY_DATA, + DATA, + "column`_'with""_quotes", + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1" +-- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql new file mode 100644 index 000000000000..d659cf82cab8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql @@ -0,0 +1,21 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID as + varchar +), '') || '-' || coalesce(cast(DOUBLE_ARRAY_DATA as + varchar +), '') || '-' || coalesce(cast(DATA as + varchar +), '') || '-' || coalesce(cast("column`_'with""_quotes" as + varchar +), '') as + varchar +)) as _AIRBYTE_PARTITION_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2" tmp +-- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql new file mode 100644 index 000000000000..17a36065ce62 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _AIRBYTE_PARTITION_HASHID, + to_varchar(get_path(parse_json("column`_'with""_quotes".value), '"currency"')) as CURRENCY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as table_alias +-- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +cross join table(flatten("column`_'with""_quotes")) as "column`_'with""_quotes" +where 1 = 1 +and "column`_'with""_quotes" is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql new file mode 100644 index 000000000000..de013bb4fb92 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _AIRBYTE_PARTITION_HASHID, + cast(CURRENCY as + varchar +) as CURRENCY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1" +-- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql new file mode 100644 index 000000000000..1996258464ce --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_AIRBYTE_PARTITION_HASHID as + varchar +), '') || '-' || coalesce(cast(CURRENCY as + varchar +), '') as + varchar +)) as _AIRBYTE_COLUMN___WITH__QUOTES_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2" tmp +-- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql new file mode 100644 index 000000000000..fbd05545c768 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _AIRBYTE_PARTITION_HASHID, + to_varchar(get_path(parse_json(DATA.value), '"currency"')) as CURRENCY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as table_alias +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +cross join table(flatten(DATA)) as DATA +where 1 = 1 +and DATA is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql new file mode 100644 index 000000000000..bdf9cd2b386c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _AIRBYTE_PARTITION_HASHID, + cast(CURRENCY as + varchar +) as CURRENCY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1" +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql new file mode 100644 index 000000000000..c4a6deea96b7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_AIRBYTE_PARTITION_HASHID as + varchar +), '') || '-' || coalesce(cast(CURRENCY as + varchar +), '') as + varchar +)) as _AIRBYTE_DATA_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2" tmp +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql new file mode 100644 index 000000000000..3013309f1f90 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _AIRBYTE_PARTITION_HASHID, + to_varchar(get_path(parse_json(DOUBLE_ARRAY_DATA.value), '"id"')) as ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as table_alias +-- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +cross join table(flatten(DOUBLE_ARRAY_DATA)) as DOUBLE_ARRAY_DATA +where 1 = 1 +and DOUBLE_ARRAY_DATA is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql new file mode 100644 index 000000000000..5c1e8b226a6a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _AIRBYTE_PARTITION_HASHID, + cast(ID as + varchar +) as ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1" +-- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql new file mode 100644 index 000000000000..662055a5ee44 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_AIRBYTE_PARTITION_HASHID as + varchar +), '') || '-' || coalesce(cast(ID as + varchar +), '') as + varchar +)) as _AIRBYTE_DOUBLE_ARRAY_DATA_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2" tmp +-- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql new file mode 100644 index 000000000000..44ffce39889f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql @@ -0,0 +1,14 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES as table_alias +-- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql new file mode 100644 index 000000000000..437075adbab4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql @@ -0,0 +1,18 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + varchar +) as ID, + cast(DATE as + varchar +) as DATE, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1" +-- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql new file mode 100644 index 000000000000..70f3a3546307 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(DATE as + varchar +), '') as + varchar +)) as _AIRBYTE_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2" tmp +-- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql new file mode 100644 index 000000000000..3c6d0a2e24d2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql @@ -0,0 +1,15 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + get_path(parse_json(_airbyte_data), '"children"') as CHILDREN, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_UNNEST_ALIAS as table_alias +-- UNNEST_ALIAS +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql new file mode 100644 index 000000000000..520bc1d1d7ec --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + bigint +) as ID, + CHILDREN, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB1" +-- UNNEST_ALIAS +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql new file mode 100644 index 000000000000..753d624ac210 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql @@ -0,0 +1,18 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(CHILDREN as + varchar +), '') as + varchar +)) as _AIRBYTE_UNNEST_ALIAS_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB2" tmp +-- UNNEST_ALIAS +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql new file mode 100644 index 000000000000..a4717707483e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql @@ -0,0 +1,20 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _AIRBYTE_UNNEST_ALIAS_HASHID, + to_varchar(get_path(parse_json(CHILDREN.value), '"ab_id"')) as AB_ID, + + get_path(parse_json(CHILDREN.value), '"owner"') + as OWNER, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS" as table_alias +-- CHILDREN at unnest_alias/children +cross join table(flatten(CHILDREN)) as CHILDREN +where 1 = 1 +and CHILDREN is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql new file mode 100644 index 000000000000..b0fd97c240a0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql @@ -0,0 +1,19 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _AIRBYTE_UNNEST_ALIAS_HASHID, + cast(AB_ID as + bigint +) as AB_ID, + cast(OWNER as + variant +) as OWNER, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB1" +-- CHILDREN at unnest_alias/children +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql new file mode 100644 index 000000000000..88b1593cc4d0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql @@ -0,0 +1,19 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_AIRBYTE_UNNEST_ALIAS_HASHID as + varchar +), '') || '-' || coalesce(cast(AB_ID as + varchar +), '') || '-' || coalesce(cast(OWNER as + varchar +), '') as + varchar +)) as _AIRBYTE_CHILDREN_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB2" tmp +-- CHILDREN at unnest_alias/children +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql new file mode 100644 index 000000000000..806d713ba2f3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql @@ -0,0 +1,15 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _AIRBYTE_CHILDREN_HASHID, + to_varchar(get_path(parse_json(OWNER), '"owner_id"')) as OWNER_ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN" as table_alias +-- OWNER at unnest_alias/children/owner +where 1 = 1 +and OWNER is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql new file mode 100644 index 000000000000..4e3f70fc2e82 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _AIRBYTE_CHILDREN_HASHID, + cast(OWNER_ID as + bigint +) as OWNER_ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB1" +-- OWNER at unnest_alias/children/owner +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql new file mode 100644 index 000000000000..79189e35ffbe --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_AIRBYTE_CHILDREN_HASHID as + varchar +), '') || '-' || coalesce(cast(OWNER_ID as + varchar +), '') as + varchar +)) as _AIRBYTE_OWNER_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB2" tmp +-- OWNER at unnest_alias/children/owner +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql new file mode 100644 index 000000000000..dc9d26c10728 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql @@ -0,0 +1,14 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION_NAMESPACE._AIRBYTE_RAW_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES as table_alias +-- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql new file mode 100644 index 000000000000..ac66f7e51acd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql @@ -0,0 +1,18 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + varchar +) as ID, + cast(DATE as + varchar +) as DATE, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1" +-- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql new file mode 100644 index 000000000000..0becb20f6e1b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(DATE as + varchar +), '') as + varchar +)) as _AIRBYTE_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2" tmp +-- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql new file mode 100644 index 000000000000..38b63ebc54d3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql @@ -0,0 +1,26 @@ +begin; + + + + + + + + merge into "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES" as DBT_INTERNAL_DEST + using "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES__dbt_tmp" as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._AIRBYTE_UNIQUE_KEY = DBT_INTERNAL_DEST._AIRBYTE_UNIQUE_KEY + + + + when matched then update set + "_AIRBYTE_UNIQUE_KEY" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY","ID" = DBT_INTERNAL_SOURCE."ID","DATE" = DBT_INTERNAL_SOURCE."DATE","PARTITION" = DBT_INTERNAL_SOURCE."PARTITION","_AIRBYTE_AB_ID" = DBT_INTERNAL_SOURCE."_AIRBYTE_AB_ID","_AIRBYTE_EMITTED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_EMITTED_AT","_AIRBYTE_NORMALIZED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_NORMALIZED_AT","_AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID" = DBT_INTERNAL_SOURCE."_AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID" + + + when not matched then insert + ("_AIRBYTE_UNIQUE_KEY", "ID", "DATE", "PARTITION", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID") + values + ("_AIRBYTE_UNIQUE_KEY", "ID", "DATE", "PARTITION", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID") + +; + commit; \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/UNNEST_ALIAS.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/UNNEST_ALIAS.sql new file mode 100644 index 000000000000..6938df462140 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/UNNEST_ALIAS.sql @@ -0,0 +1,26 @@ +begin; + + + + + + + + merge into "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS" as DBT_INTERNAL_DEST + using "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS__dbt_tmp" as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._AIRBYTE_AB_ID = DBT_INTERNAL_DEST._AIRBYTE_AB_ID + + + + when matched then update set + "ID" = DBT_INTERNAL_SOURCE."ID","CHILDREN" = DBT_INTERNAL_SOURCE."CHILDREN","_AIRBYTE_AB_ID" = DBT_INTERNAL_SOURCE."_AIRBYTE_AB_ID","_AIRBYTE_EMITTED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_EMITTED_AT","_AIRBYTE_NORMALIZED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_NORMALIZED_AT","_AIRBYTE_UNNEST_ALIAS_HASHID" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNNEST_ALIAS_HASHID" + + + when not matched then insert + ("ID", "CHILDREN", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_UNNEST_ALIAS_HASHID") + values + ("ID", "CHILDREN", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_UNNEST_ALIAS_HASHID") + +; + commit; \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql new file mode 100644 index 000000000000..cbca5b47ca42 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql @@ -0,0 +1,26 @@ +begin; + + + + + + + + merge into "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD" as DBT_INTERNAL_DEST + using "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD__dbt_tmp" as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._AIRBYTE_UNIQUE_KEY_SCD = DBT_INTERNAL_DEST._AIRBYTE_UNIQUE_KEY_SCD + + + + when matched then update set + "_AIRBYTE_UNIQUE_KEY" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY","_AIRBYTE_UNIQUE_KEY_SCD" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY_SCD","ID" = DBT_INTERNAL_SOURCE."ID","DATE" = DBT_INTERNAL_SOURCE."DATE","PARTITION" = DBT_INTERNAL_SOURCE."PARTITION","_AIRBYTE_START_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_START_AT","_AIRBYTE_END_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_END_AT","_AIRBYTE_ACTIVE_ROW" = DBT_INTERNAL_SOURCE."_AIRBYTE_ACTIVE_ROW","_AIRBYTE_AB_ID" = DBT_INTERNAL_SOURCE."_AIRBYTE_AB_ID","_AIRBYTE_EMITTED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_EMITTED_AT","_AIRBYTE_NORMALIZED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_NORMALIZED_AT","_AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID" = DBT_INTERNAL_SOURCE."_AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID" + + + when not matched then insert + ("_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_UNIQUE_KEY_SCD", "ID", "DATE", "PARTITION", "_AIRBYTE_START_AT", "_AIRBYTE_END_AT", "_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID") + values + ("_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_UNIQUE_KEY_SCD", "ID", "DATE", "PARTITION", "_AIRBYTE_START_AT", "_AIRBYTE_END_AT", "_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID") + +; + commit; \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql new file mode 100644 index 000000000000..a366f2bba8cc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY" as + (select * from( + +-- Final base SQL model +select + ID, + CONFLICT_STREAM_ARRAY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_CONFLICT_STREAM_ARRAY_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB3" +-- CONFLICT_STREAM_ARRAY from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_ARRAY +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql new file mode 100644 index 000000000000..a20080a7f7f4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME" as + (select * from( + +-- Final base SQL model +select + ID, + CONFLICT_STREAM_NAME, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_CONFLICT_STREAM_NAME_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB3" +-- CONFLICT_STREAM_NAME from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_NAME +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql new file mode 100644 index 000000000000..4d69fa730172 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, + CONFLICT_STREAM_NAME, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3" +-- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name from "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql new file mode 100644 index 000000000000..97d25955385f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, + GROUPS, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_CONFLICT_STREAM_NAME_3_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3" +-- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name from "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql new file mode 100644 index 000000000000..efc7cbe9dce6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR" as + (select * from( + +-- Final base SQL model +select + ID, + CONFLICT_STREAM_SCALAR, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_CONFLICT_STREAM_SCALAR_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB3" +-- CONFLICT_STREAM_SCALAR from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_SCALAR +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql new file mode 100644 index 000000000000..75228a851232 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql @@ -0,0 +1,21 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, + DOUBLE_ARRAY_DATA, + DATA, + "column`_'with""_quotes", + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_PARTITION_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3" +-- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql new file mode 100644 index 000000000000..1f2adeefe899 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_PARTITION_HASHID, + CURRENCY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_COLUMN___WITH__QUOTES_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3" +-- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql new file mode 100644 index 000000000000..cfc1ce750875 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_PARTITION_HASHID, + CURRENCY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_DATA_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3" +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql new file mode 100644 index 000000000000..4bdef077e4b3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_PARTITION_HASHID, + ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_DOUBLE_ARRAY_DATA_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3" +-- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql new file mode 100644 index 000000000000..a470f14eceac --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES" as + (select * from( + +-- Final base SQL model +select + ID, + DATE, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3" +-- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql new file mode 100644 index 000000000000..0657b14de313 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql @@ -0,0 +1,20 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_UNNEST_ALIAS_HASHID, + AB_ID, + OWNER, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_CHILDREN_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB3" +-- CHILDREN at unnest_alias/children from "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql new file mode 100644 index 000000000000..652a9bd48931 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_CHILDREN_HASHID, + OWNER_ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_OWNER_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB3" +-- OWNER at unnest_alias/children/owner from "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql new file mode 100644 index 000000000000..057fea2c9ee7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES" as + (select * from( + +-- Final base SQL model +select + ID, + DATE, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3" +-- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES from "AIRBYTE_DATABASE".TEST_NORMALIZATION_NAMESPACE._AIRBYTE_RAW_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql new file mode 100644 index 000000000000..b224c3ada40e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql @@ -0,0 +1,20 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(DATE as + varchar +), '') || '-' || coalesce(cast(PARTITION as + varchar +), '') as + varchar +)) as _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2" tmp +-- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql deleted file mode 100644 index 4ddfa66b4825..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql +++ /dev/null @@ -1,15 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - to_varchar(get_path(parse_json(_airbyte_data), '"name"')) as NAME, - to_varchar(get_path(parse_json(_airbyte_data), '"column`_''with""_quotes"')) as "column`_'with""_quotes", - to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_lsn"')) as _AB_CDC_LSN, - to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_updated_at"')) as _AB_CDC_UPDATED_AT, - to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_deleted_at"')) as _AB_CDC_DELETED_AT, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_CDC_EXCLUDED as table_alias --- DEDUP_CDC_EXCLUDED - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql deleted file mode 100644 index dd52fb86ff3f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql +++ /dev/null @@ -1,27 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - bigint -) as ID, - cast(NAME as - varchar -) as NAME, - cast("column`_'with""_quotes" as - varchar -) as "column`_'with""_quotes", - cast(_AB_CDC_LSN as - float -) as _AB_CDC_LSN, - cast(_AB_CDC_UPDATED_AT as - float -) as _AB_CDC_UPDATED_AT, - cast(_AB_CDC_DELETED_AT as - float -) as _AB_CDC_DELETED_AT, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB1" --- DEDUP_CDC_EXCLUDED - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql deleted file mode 100644 index a4b8a3aedf2f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql +++ /dev/null @@ -1,24 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') || '-' || coalesce(cast(NAME as - varchar -), '') || '-' || coalesce(cast("column`_'with""_quotes" as - varchar -), '') || '-' || coalesce(cast(_AB_CDC_LSN as - varchar -), '') || '-' || coalesce(cast(_AB_CDC_UPDATED_AT as - varchar -), '') || '-' || coalesce(cast(_AB_CDC_DELETED_AT as - varchar -), '') as - varchar -)) as _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB2" tmp --- DEDUP_CDC_EXCLUDED - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB4.sql deleted file mode 100644 index c52930b6182a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB4.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB4" as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID - order by _AIRBYTE_EMITTED_AT asc - ) as _AIRBYTE_ROW_NUM, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB3" tmp --- DEDUP_CDC_EXCLUDED from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_CDC_EXCLUDED - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql deleted file mode 100644 index 124336cd315d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - to_varchar(get_path(parse_json(_airbyte_data), '"currency"')) as CURRENCY, - to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, - to_varchar(get_path(parse_json(_airbyte_data), '"timestamp_col"')) as TIMESTAMP_COL, - to_varchar(get_path(parse_json(_airbyte_data), '"HKD@spéçiäl & characters"')) as "HKD@spéçiäl & characters", - to_varchar(get_path(parse_json(_airbyte_data), '"HKD_special___characters"')) as HKD_SPECIAL___CHARACTERS, - to_varchar(get_path(parse_json(_airbyte_data), '"NZD"')) as NZD, - to_varchar(get_path(parse_json(_airbyte_data), '"USD"')) as USD, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_EXCHANGE_RATE as table_alias --- DEDUP_EXCHANGE_RATE - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql deleted file mode 100644 index 7b76d03348ee..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql +++ /dev/null @@ -1,39 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - bigint -) as ID, - cast(CURRENCY as - varchar -) as CURRENCY, - cast(nullif(DATE, '') as - date -) as DATE, - case - when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZHTZM') - when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZH') - when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZHTZM') - when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZH') - when TIMESTAMP_COL = '' then NULL - else to_timestamp_tz(TIMESTAMP_COL) - end as TIMESTAMP_COL - , - cast("HKD@spéçiäl & characters" as - float -) as "HKD@spéçiäl & characters", - cast(HKD_SPECIAL___CHARACTERS as - varchar -) as HKD_SPECIAL___CHARACTERS, - cast(NZD as - float -) as NZD, - cast(USD as - float -) as USD, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB1" --- DEDUP_EXCHANGE_RATE - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql deleted file mode 100644 index 9049fa459fe0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql +++ /dev/null @@ -1,28 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') || '-' || coalesce(cast(CURRENCY as - varchar -), '') || '-' || coalesce(cast(DATE as - varchar -), '') || '-' || coalesce(cast(TIMESTAMP_COL as - varchar -), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as - varchar -), '') || '-' || coalesce(cast(HKD_SPECIAL___CHARACTERS as - varchar -), '') || '-' || coalesce(cast(NZD as - varchar -), '') || '-' || coalesce(cast(USD as - varchar -), '') as - varchar -)) as _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB2" tmp --- DEDUP_EXCHANGE_RATE - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB4.sql deleted file mode 100644 index 6205e9e1fbd1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB4.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB4" as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID - order by _AIRBYTE_EMITTED_AT asc - ) as _AIRBYTE_ROW_NUM, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB3" tmp --- DEDUP_EXCHANGE_RATE from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_EXCHANGE_RATE - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql deleted file mode 100644 index 427d37de5377..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - to_varchar(get_path(parse_json(_airbyte_data), '"currency"')) as CURRENCY, - to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, - to_varchar(get_path(parse_json(_airbyte_data), '"timestamp_col"')) as TIMESTAMP_COL, - to_varchar(get_path(parse_json(_airbyte_data), '"HKD@spéçiäl & characters"')) as "HKD@spéçiäl & characters", - to_varchar(get_path(parse_json(_airbyte_data), '"HKD_special___characters"')) as HKD_SPECIAL___CHARACTERS, - to_varchar(get_path(parse_json(_airbyte_data), '"NZD"')) as NZD, - to_varchar(get_path(parse_json(_airbyte_data), '"USD"')) as USD, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_EXCHANGE_RATE as table_alias --- EXCHANGE_RATE - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql deleted file mode 100644 index d9459cd75f1b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql +++ /dev/null @@ -1,39 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - bigint -) as ID, - cast(CURRENCY as - varchar -) as CURRENCY, - cast(nullif(DATE, '') as - date -) as DATE, - case - when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZHTZM') - when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZH') - when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZHTZM') - when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZH') - when TIMESTAMP_COL = '' then NULL - else to_timestamp_tz(TIMESTAMP_COL) - end as TIMESTAMP_COL - , - cast("HKD@spéçiäl & characters" as - float -) as "HKD@spéçiäl & characters", - cast(HKD_SPECIAL___CHARACTERS as - varchar -) as HKD_SPECIAL___CHARACTERS, - cast(NZD as - float -) as NZD, - cast(USD as - float -) as USD, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB1" --- EXCHANGE_RATE - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql deleted file mode 100644 index c69cb6fc10b9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql +++ /dev/null @@ -1,28 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') || '-' || coalesce(cast(CURRENCY as - varchar -), '') || '-' || coalesce(cast(DATE as - varchar -), '') || '-' || coalesce(cast(TIMESTAMP_COL as - varchar -), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as - varchar -), '') || '-' || coalesce(cast(HKD_SPECIAL___CHARACTERS as - varchar -), '') || '-' || coalesce(cast(NZD as - varchar -), '') || '-' || coalesce(cast(USD as - varchar -), '') as - varchar -)) as _AIRBYTE_EXCHANGE_RATE_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB2" tmp --- EXCHANGE_RATE - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql deleted file mode 100644 index fb06f48abfe5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql +++ /dev/null @@ -1,15 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - to_varchar(get_path(parse_json(_airbyte_data), '"name"')) as NAME, - to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_lsn"')) as _AB_CDC_LSN, - to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_updated_at"')) as _AB_CDC_UPDATED_AT, - to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_deleted_at"')) as _AB_CDC_DELETED_AT, - to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_log_pos"')) as _AB_CDC_LOG_POS, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_POS_DEDUP_CDCX as table_alias --- POS_DEDUP_CDCX - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql deleted file mode 100644 index 0bd1e24eae02..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql +++ /dev/null @@ -1,27 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - bigint -) as ID, - cast(NAME as - varchar -) as NAME, - cast(_AB_CDC_LSN as - float -) as _AB_CDC_LSN, - cast(_AB_CDC_UPDATED_AT as - float -) as _AB_CDC_UPDATED_AT, - cast(_AB_CDC_DELETED_AT as - float -) as _AB_CDC_DELETED_AT, - cast(_AB_CDC_LOG_POS as - float -) as _AB_CDC_LOG_POS, - _AIRBYTE_EMITTED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB1" --- POS_DEDUP_CDCX - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql deleted file mode 100644 index 5b8827c33977..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql +++ /dev/null @@ -1,24 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') || '-' || coalesce(cast(NAME as - varchar -), '') || '-' || coalesce(cast(_AB_CDC_LSN as - varchar -), '') || '-' || coalesce(cast(_AB_CDC_UPDATED_AT as - varchar -), '') || '-' || coalesce(cast(_AB_CDC_DELETED_AT as - varchar -), '') || '-' || coalesce(cast(_AB_CDC_LOG_POS as - varchar -), '') as - varchar -)) as _AIRBYTE_POS_DEDUP_CDCX_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB2" tmp --- POS_DEDUP_CDCX - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB4.sql deleted file mode 100644 index 6b378a26afcf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB4.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB4" as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _AIRBYTE_POS_DEDUP_CDCX_HASHID - order by _AIRBYTE_EMITTED_AT asc - ) as _AIRBYTE_ROW_NUM, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB3" tmp --- POS_DEDUP_CDCX from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_POS_DEDUP_CDCX - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql deleted file mode 100644 index a530c9d73be3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED" as - ( --- Final base SQL model -select - ID, - NAME, - "column`_'with""_quotes", - _AB_CDC_LSN, - _AB_CDC_UPDATED_AT, - _AB_CDC_DELETED_AT, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_SCD" --- DEDUP_CDC_EXCLUDED from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_CDC_EXCLUDED -where _airbyte_active_row = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql deleted file mode 100644 index e083558a535d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql +++ /dev/null @@ -1,28 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_SCD" as - ( --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - ID, - NAME, - "column`_'with""_quotes", - _AB_CDC_LSN, - _AB_CDC_UPDATED_AT, - _AB_CDC_DELETED_AT, - _AIRBYTE_EMITTED_AT as _AIRBYTE_START_AT, - lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by _AIRBYTE_EMITTED_AT is null asc, _AIRBYTE_EMITTED_AT desc, _AIRBYTE_EMITTED_AT desc - ) as _AIRBYTE_END_AT, - case when lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by _AIRBYTE_EMITTED_AT is null asc, _AIRBYTE_EMITTED_AT desc, _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc - ) is null and _AB_CDC_DELETED_AT is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB4" --- DEDUP_CDC_EXCLUDED from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_CDC_EXCLUDED -where _airbyte_row_num = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql deleted file mode 100644 index 89d7840d3a68..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE" as - ( --- Final base SQL model -select - ID, - CURRENCY, - DATE, - TIMESTAMP_COL, - "HKD@spéçiäl & characters", - HKD_SPECIAL___CHARACTERS, - NZD, - USD, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_SCD" --- DEDUP_EXCHANGE_RATE from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_EXCHANGE_RATE -where _airbyte_active_row = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql deleted file mode 100644 index 87d94b17d25c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql +++ /dev/null @@ -1,34 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_SCD" as - ( --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - ID, - CURRENCY, - DATE, - TIMESTAMP_COL, - "HKD@spéçiäl & characters", - HKD_SPECIAL___CHARACTERS, - NZD, - USD, - DATE as _AIRBYTE_START_AT, - lag(DATE) over ( - partition by ID, CURRENCY, cast(NZD as - varchar -) - order by DATE is null asc, DATE desc, _AIRBYTE_EMITTED_AT desc - ) as _AIRBYTE_END_AT, - case when lag(DATE) over ( - partition by ID, CURRENCY, cast(NZD as - varchar -) - order by DATE is null asc, DATE desc, _AIRBYTE_EMITTED_AT desc - ) is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB4" --- DEDUP_EXCHANGE_RATE from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_EXCHANGE_RATE -where _airbyte_row_num = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql deleted file mode 100644 index b669de625a2b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."EXCHANGE_RATE" as - ( --- Final base SQL model -select - ID, - CURRENCY, - DATE, - TIMESTAMP_COL, - "HKD@spéçiäl & characters", - HKD_SPECIAL___CHARACTERS, - NZD, - USD, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_EXCHANGE_RATE_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB3" --- EXCHANGE_RATE from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_EXCHANGE_RATE - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql deleted file mode 100644 index 8892c9c7bda7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX" as - ( --- Final base SQL model -select - ID, - NAME, - _AB_CDC_LSN, - _AB_CDC_UPDATED_AT, - _AB_CDC_DELETED_AT, - _AB_CDC_LOG_POS, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_POS_DEDUP_CDCX_HASHID -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX_SCD" --- POS_DEDUP_CDCX from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_POS_DEDUP_CDCX -where _airbyte_active_row = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql deleted file mode 100644 index 292013c109cb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql +++ /dev/null @@ -1,28 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX_SCD" as - ( --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - ID, - NAME, - _AB_CDC_LSN, - _AB_CDC_UPDATED_AT, - _AB_CDC_DELETED_AT, - _AB_CDC_LOG_POS, - _AIRBYTE_EMITTED_AT as _AIRBYTE_START_AT, - lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by _AIRBYTE_EMITTED_AT is null asc, _AIRBYTE_EMITTED_AT desc, _AIRBYTE_EMITTED_AT desc - ) as _AIRBYTE_END_AT, - case when lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by _AIRBYTE_EMITTED_AT is null asc, _AIRBYTE_EMITTED_AT desc, _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc, _AB_CDC_LOG_POS desc - ) is null and _AB_CDC_DELETED_AT is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_POS_DEDUP_CDCX_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB4" --- POS_DEDUP_CDCX from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_POS_DEDUP_CDCX -where _airbyte_row_num = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql new file mode 100644 index 000000000000..8d5282887fb0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql @@ -0,0 +1,18 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"name"')) as NAME, + to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_lsn"')) as _AB_CDC_LSN, + to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_updated_at"')) as _AB_CDC_UPDATED_AT, + to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_deleted_at"')) as _AB_CDC_DELETED_AT, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_CDC_EXCLUDED as table_alias +-- DEDUP_CDC_EXCLUDED +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql new file mode 100644 index 000000000000..57a50d43e5bc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql @@ -0,0 +1,28 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + bigint +) as ID, + cast(NAME as + varchar +) as NAME, + cast(_AB_CDC_LSN as + float +) as _AB_CDC_LSN, + cast(_AB_CDC_UPDATED_AT as + float +) as _AB_CDC_UPDATED_AT, + cast(_AB_CDC_DELETED_AT as + float +) as _AB_CDC_DELETED_AT, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB1" +-- DEDUP_CDC_EXCLUDED +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql new file mode 100644 index 000000000000..ee207b768a20 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql @@ -0,0 +1,21 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"currency"')) as CURRENCY, + to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, + to_varchar(get_path(parse_json(_airbyte_data), '"timestamp_col"')) as TIMESTAMP_COL, + to_varchar(get_path(parse_json(_airbyte_data), '"HKD@spéçiäl & characters"')) as "HKD@spéçiäl & characters", + to_varchar(get_path(parse_json(_airbyte_data), '"HKD_special___characters"')) as HKD_SPECIAL___CHARACTERS, + to_varchar(get_path(parse_json(_airbyte_data), '"NZD"')) as NZD, + to_varchar(get_path(parse_json(_airbyte_data), '"USD"')) as USD, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_EXCHANGE_RATE as table_alias +-- DEDUP_EXCHANGE_RATE +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql new file mode 100644 index 000000000000..66b09ec2508e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql @@ -0,0 +1,43 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + bigint +) as ID, + cast(CURRENCY as + varchar +) as CURRENCY, + cast(nullif(DATE, '') as + date +) as DATE, + case + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZHTZM') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZH') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZHTZM') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZH') + when TIMESTAMP_COL = '' then NULL + else to_timestamp_tz(TIMESTAMP_COL) + end as TIMESTAMP_COL + , + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(HKD_SPECIAL___CHARACTERS as + varchar +) as HKD_SPECIAL___CHARACTERS, + cast(NZD as + float +) as NZD, + cast(USD as + float +) as USD, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB1" +-- DEDUP_EXCHANGE_RATE +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql new file mode 100644 index 000000000000..33e07a844a1c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql @@ -0,0 +1,21 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"currency"')) as CURRENCY, + to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, + to_varchar(get_path(parse_json(_airbyte_data), '"timestamp_col"')) as TIMESTAMP_COL, + to_varchar(get_path(parse_json(_airbyte_data), '"HKD@spéçiäl & characters"')) as "HKD@spéçiäl & characters", + to_varchar(get_path(parse_json(_airbyte_data), '"HKD_special___characters"')) as HKD_SPECIAL___CHARACTERS, + to_varchar(get_path(parse_json(_airbyte_data), '"NZD"')) as NZD, + to_varchar(get_path(parse_json(_airbyte_data), '"USD"')) as USD, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_EXCHANGE_RATE as table_alias +-- EXCHANGE_RATE +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql new file mode 100644 index 000000000000..f5ef6e473984 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql @@ -0,0 +1,43 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + bigint +) as ID, + cast(CURRENCY as + varchar +) as CURRENCY, + cast(nullif(DATE, '') as + date +) as DATE, + case + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZHTZM') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZH') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZHTZM') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZH') + when TIMESTAMP_COL = '' then NULL + else to_timestamp_tz(TIMESTAMP_COL) + end as TIMESTAMP_COL + , + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(HKD_SPECIAL___CHARACTERS as + varchar +) as HKD_SPECIAL___CHARACTERS, + cast(NZD as + float +) as NZD, + cast(USD as + float +) as USD, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB1" +-- EXCHANGE_RATE +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql new file mode 100644 index 000000000000..c1f0840c3677 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql @@ -0,0 +1,30 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(CURRENCY as + varchar +), '') || '-' || coalesce(cast(DATE as + varchar +), '') || '-' || coalesce(cast(TIMESTAMP_COL as + varchar +), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as + varchar +), '') || '-' || coalesce(cast(HKD_SPECIAL___CHARACTERS as + varchar +), '') || '-' || coalesce(cast(NZD as + varchar +), '') || '-' || coalesce(cast(USD as + varchar +), '') as + varchar +)) as _AIRBYTE_EXCHANGE_RATE_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB2" tmp +-- EXCHANGE_RATE +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql new file mode 100644 index 000000000000..93d03ade5063 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql @@ -0,0 +1,18 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"name"')) as NAME, + to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_lsn"')) as _AB_CDC_LSN, + to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_updated_at"')) as _AB_CDC_UPDATED_AT, + to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_deleted_at"')) as _AB_CDC_DELETED_AT, + to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_log_pos"')) as _AB_CDC_LOG_POS, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_POS_DEDUP_CDCX as table_alias +-- POS_DEDUP_CDCX +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql new file mode 100644 index 000000000000..82b6ca1c2619 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql @@ -0,0 +1,30 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + bigint +) as ID, + cast(NAME as + varchar +) as NAME, + cast(_AB_CDC_LSN as + float +) as _AB_CDC_LSN, + cast(_AB_CDC_UPDATED_AT as + float +) as _AB_CDC_UPDATED_AT, + cast(_AB_CDC_DELETED_AT as + float +) as _AB_CDC_DELETED_AT, + cast(_AB_CDC_LOG_POS as + float +) as _AB_CDC_LOG_POS, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB1" +-- POS_DEDUP_CDCX +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql new file mode 100644 index 000000000000..123971c4e5dc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql @@ -0,0 +1,14 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_RENAMED_DEDUP_CDC_EXCLUDED as table_alias +-- RENAMED_DEDUP_CDC_EXCLUDED +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql new file mode 100644 index 000000000000..35438f37c247 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + bigint +) as ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB1" +-- RENAMED_DEDUP_CDC_EXCLUDED +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql new file mode 100644 index 000000000000..08e7e20215a1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql @@ -0,0 +1,25 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_UNIQUE_KEY, + ID, + NAME, + _AB_CDC_LSN, + _AB_CDC_UPDATED_AT, + _AB_CDC_DELETED_AT, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_SCD" +-- DEDUP_CDC_EXCLUDED from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_CDC_EXCLUDED +where 1 = 1 +and _AIRBYTE_ACTIVE_ROW = 1 + + ) order by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED" cluster by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql new file mode 100644 index 000000000000..1a9c41c14890 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql @@ -0,0 +1,28 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_UNIQUE_KEY, + ID, + CURRENCY, + DATE, + TIMESTAMP_COL, + "HKD@spéçiäl & characters", + HKD_SPECIAL___CHARACTERS, + NZD, + USD, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_SCD" +-- DEDUP_EXCHANGE_RATE from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_EXCHANGE_RATE +where 1 = 1 +and _AIRBYTE_ACTIVE_ROW = 1 + + ) order by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE" cluster by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/EXCHANGE_RATE.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/EXCHANGE_RATE.sql new file mode 100644 index 000000000000..3420de9fa251 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/EXCHANGE_RATE.sql @@ -0,0 +1,26 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."EXCHANGE_RATE" as + (select * from( + +-- Final base SQL model +select + ID, + CURRENCY, + DATE, + TIMESTAMP_COL, + "HKD@spéçiäl & characters", + HKD_SPECIAL___CHARACTERS, + NZD, + USD, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_EXCHANGE_RATE_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB3" +-- EXCHANGE_RATE from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_EXCHANGE_RATE +where 1 = 1 + + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."EXCHANGE_RATE" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql new file mode 100644 index 000000000000..f5c3fb018edc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql @@ -0,0 +1,21 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_UNIQUE_KEY, + ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_SCD" +-- RENAMED_DEDUP_CDC_EXCLUDED from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_RENAMED_DEDUP_CDC_EXCLUDED +where 1 = 1 +and _AIRBYTE_ACTIVE_ROW = 1 + + ) order by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED" cluster by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql new file mode 100644 index 000000000000..343251eb8988 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql @@ -0,0 +1,93 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_SCD" as + (select * from( + +with + +input_data as ( + select * + from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB3" + -- DEDUP_CDC_EXCLUDED from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_CDC_EXCLUDED +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast(ID as + varchar +), '') as + varchar +)) as _AIRBYTE_UNIQUE_KEY, + ID, + NAME, + _AB_CDC_LSN, + _AB_CDC_UPDATED_AT, + _AB_CDC_DELETED_AT, + _AIRBYTE_EMITTED_AT as _AIRBYTE_START_AT, + lag(_AIRBYTE_EMITTED_AT) over ( + partition by ID + order by + _AIRBYTE_EMITTED_AT is null asc, + _AIRBYTE_EMITTED_AT desc, + _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc + ) as _AIRBYTE_END_AT, + case when lag(_AIRBYTE_EMITTED_AT) over ( + partition by ID + order by + _AIRBYTE_EMITTED_AT is null asc, + _AIRBYTE_EMITTED_AT desc, + _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc + ) is null and _AB_CDC_DELETED_AT is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT, cast(_AB_CDC_DELETED_AT as + varchar +), cast(_AB_CDC_UPDATED_AT as + varchar +) + order by _AIRBYTE_AB_ID + ) as _AIRBYTE_ROW_NUM, + md5(cast(coalesce(cast(_AIRBYTE_UNIQUE_KEY as + varchar +), '') || '-' || coalesce(cast(_AIRBYTE_START_AT as + varchar +), '') || '-' || coalesce(cast(_AIRBYTE_EMITTED_AT as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_DELETED_AT as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_UPDATED_AT as + varchar +), '') as + varchar +)) as _AIRBYTE_UNIQUE_KEY_SCD, + scd_data.* + from scd_data +) +select + _AIRBYTE_UNIQUE_KEY, + _AIRBYTE_UNIQUE_KEY_SCD, + ID, + NAME, + _AB_CDC_LSN, + _AB_CDC_UPDATED_AT, + _AB_CDC_DELETED_AT, + _AIRBYTE_START_AT, + _AIRBYTE_END_AT, + _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID +from dedup_data where _AIRBYTE_ROW_NUM = 1 + ) order by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_SCD" cluster by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql new file mode 100644 index 000000000000..f1a29aefb339 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql @@ -0,0 +1,99 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_SCD" as + (select * from( + +with + +input_data as ( + select * + from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB3" + -- DEDUP_EXCHANGE_RATE from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_EXCHANGE_RATE +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(CURRENCY as + varchar +), '') || '-' || coalesce(cast(NZD as + varchar +), '') as + varchar +)) as _AIRBYTE_UNIQUE_KEY, + ID, + CURRENCY, + DATE, + TIMESTAMP_COL, + "HKD@spéçiäl & characters", + HKD_SPECIAL___CHARACTERS, + NZD, + USD, + DATE as _AIRBYTE_START_AT, + lag(DATE) over ( + partition by ID, CURRENCY, cast(NZD as + varchar +) + order by + DATE is null asc, + DATE desc, + _AIRBYTE_EMITTED_AT desc + ) as _AIRBYTE_END_AT, + case when lag(DATE) over ( + partition by ID, CURRENCY, cast(NZD as + varchar +) + order by + DATE is null asc, + DATE desc, + _AIRBYTE_EMITTED_AT desc + ) is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT + order by _AIRBYTE_AB_ID + ) as _AIRBYTE_ROW_NUM, + md5(cast(coalesce(cast(_AIRBYTE_UNIQUE_KEY as + varchar +), '') || '-' || coalesce(cast(_AIRBYTE_START_AT as + varchar +), '') || '-' || coalesce(cast(_AIRBYTE_EMITTED_AT as + varchar +), '') as + varchar +)) as _AIRBYTE_UNIQUE_KEY_SCD, + scd_data.* + from scd_data +) +select + _AIRBYTE_UNIQUE_KEY, + _AIRBYTE_UNIQUE_KEY_SCD, + ID, + CURRENCY, + DATE, + TIMESTAMP_COL, + "HKD@spéçiäl & characters", + HKD_SPECIAL___CHARACTERS, + NZD, + USD, + _AIRBYTE_START_AT, + _AIRBYTE_END_AT, + _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID +from dedup_data where _AIRBYTE_ROW_NUM = 1 + ) order by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_SCD" cluster by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql new file mode 100644 index 000000000000..c171a978fe39 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql @@ -0,0 +1,77 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_SCD" as + (select * from( + +with + +input_data as ( + select * + from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB3" + -- RENAMED_DEDUP_CDC_EXCLUDED from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_RENAMED_DEDUP_CDC_EXCLUDED +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast(ID as + varchar +), '') as + varchar +)) as _AIRBYTE_UNIQUE_KEY, + ID, + _AIRBYTE_EMITTED_AT as _AIRBYTE_START_AT, + lag(_AIRBYTE_EMITTED_AT) over ( + partition by ID + order by + _AIRBYTE_EMITTED_AT is null asc, + _AIRBYTE_EMITTED_AT desc, + _AIRBYTE_EMITTED_AT desc + ) as _AIRBYTE_END_AT, + case when lag(_AIRBYTE_EMITTED_AT) over ( + partition by ID + order by + _AIRBYTE_EMITTED_AT is null asc, + _AIRBYTE_EMITTED_AT desc, + _AIRBYTE_EMITTED_AT desc + ) is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + _AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT + order by _AIRBYTE_AB_ID + ) as _AIRBYTE_ROW_NUM, + md5(cast(coalesce(cast(_AIRBYTE_UNIQUE_KEY as + varchar +), '') || '-' || coalesce(cast(_AIRBYTE_START_AT as + varchar +), '') || '-' || coalesce(cast(_AIRBYTE_EMITTED_AT as + varchar +), '') as + varchar +)) as _AIRBYTE_UNIQUE_KEY_SCD, + scd_data.* + from scd_data +) +select + _AIRBYTE_UNIQUE_KEY, + _AIRBYTE_UNIQUE_KEY_SCD, + ID, + _AIRBYTE_START_AT, + _AIRBYTE_END_AT, + _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID +from dedup_data where _AIRBYTE_ROW_NUM = 1 + ) order by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_SCD" cluster by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql new file mode 100644 index 000000000000..198a2627ab68 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql @@ -0,0 +1,25 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_UNIQUE_KEY, + ID, + NAME, + _AB_CDC_LSN, + _AB_CDC_UPDATED_AT, + _AB_CDC_DELETED_AT, + _AB_CDC_LOG_POS, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_POS_DEDUP_CDCX_HASHID +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX_SCD" +-- POS_DEDUP_CDCX from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_POS_DEDUP_CDCX +where 1 = 1 +and _AIRBYTE_ACTIVE_ROW = 1 + ) order by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX" cluster by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql new file mode 100644 index 000000000000..24b58ed5bf4d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql @@ -0,0 +1,99 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX_SCD" as + (select * from( + +with + +input_data as ( + select * + from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB3" + -- POS_DEDUP_CDCX from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_POS_DEDUP_CDCX +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast(ID as + varchar +), '') as + varchar +)) as _AIRBYTE_UNIQUE_KEY, + ID, + NAME, + _AB_CDC_LSN, + _AB_CDC_UPDATED_AT, + _AB_CDC_DELETED_AT, + _AB_CDC_LOG_POS, + _AIRBYTE_EMITTED_AT as _AIRBYTE_START_AT, + lag(_AIRBYTE_EMITTED_AT) over ( + partition by ID + order by + _AIRBYTE_EMITTED_AT is null asc, + _AIRBYTE_EMITTED_AT desc, + _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc, _AB_CDC_LOG_POS desc + ) as _AIRBYTE_END_AT, + case when lag(_AIRBYTE_EMITTED_AT) over ( + partition by ID + order by + _AIRBYTE_EMITTED_AT is null asc, + _AIRBYTE_EMITTED_AT desc, + _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc, _AB_CDC_LOG_POS desc + ) is null and _AB_CDC_DELETED_AT is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + _AIRBYTE_POS_DEDUP_CDCX_HASHID + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT, cast(_AB_CDC_DELETED_AT as + varchar +), cast(_AB_CDC_UPDATED_AT as + varchar +), cast(_AB_CDC_LOG_POS as + varchar +) + order by _AIRBYTE_AB_ID + ) as _AIRBYTE_ROW_NUM, + md5(cast(coalesce(cast(_AIRBYTE_UNIQUE_KEY as + varchar +), '') || '-' || coalesce(cast(_AIRBYTE_START_AT as + varchar +), '') || '-' || coalesce(cast(_AIRBYTE_EMITTED_AT as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_DELETED_AT as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_UPDATED_AT as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_LOG_POS as + varchar +), '') as + varchar +)) as _AIRBYTE_UNIQUE_KEY_SCD, + scd_data.* + from scd_data +) +select + _AIRBYTE_UNIQUE_KEY, + _AIRBYTE_UNIQUE_KEY_SCD, + ID, + NAME, + _AB_CDC_LSN, + _AB_CDC_UPDATED_AT, + _AB_CDC_DELETED_AT, + _AB_CDC_LOG_POS, + _AIRBYTE_START_AT, + _AIRBYTE_END_AT, + _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_POS_DEDUP_CDCX_HASHID +from dedup_data where _AIRBYTE_ROW_NUM = 1 + ) order by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX_SCD" cluster by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql new file mode 100644 index 000000000000..522991af25cd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql @@ -0,0 +1,24 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(NAME as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_LSN as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_UPDATED_AT as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_DELETED_AT as + varchar +), '') as + varchar +)) as _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB2" tmp +-- DEDUP_CDC_EXCLUDED +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql new file mode 100644 index 000000000000..0b361061abf3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql @@ -0,0 +1,30 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(CURRENCY as + varchar +), '') || '-' || coalesce(cast(DATE as + varchar +), '') || '-' || coalesce(cast(TIMESTAMP_COL as + varchar +), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as + varchar +), '') || '-' || coalesce(cast(HKD_SPECIAL___CHARACTERS as + varchar +), '') || '-' || coalesce(cast(NZD as + varchar +), '') || '-' || coalesce(cast(USD as + varchar +), '') as + varchar +)) as _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB2" tmp +-- DEDUP_EXCHANGE_RATE +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql new file mode 100644 index 000000000000..c1189b3c1dfd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql @@ -0,0 +1,25 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(NAME as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_LSN as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_UPDATED_AT as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_DELETED_AT as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_LOG_POS as + varchar +), '') as + varchar +)) as _AIRBYTE_POS_DEDUP_CDCX_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB2" tmp +-- POS_DEDUP_CDCX +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql new file mode 100644 index 000000000000..624b8d286061 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') as + varchar +)) as _AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB2" tmp +-- RENAMED_DEDUP_CDC_EXCLUDED +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql deleted file mode 100644 index b5ed6a14b8ad..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql +++ /dev/null @@ -1,43 +0,0 @@ -with table_row_counts as ( - select distinct count(*) as row_count, 10 as expected_count - from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_EXCHANGE_RATE') }} -union all - select distinct count(*) as row_count, 10 as expected_count - from {{ ref('EXCHANGE_RATE') }} - -union all - - select distinct count(*) as row_count, 10 as expected_count - from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_EXCHANGE_RATE') }} -union all - select distinct count(*) as row_count, 10 as expected_count - from {{ ref('DEDUP_EXCHANGE_RATE_SCD') }} -union all - select distinct count(*) as row_count, 5 as expected_count - from {{ ref('DEDUP_EXCHANGE_RATE') }} - -union all - - select distinct count(*) as row_count, 8 as expected_count - from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_CDC_EXCLUDED') }} -union all - select distinct count(*) as row_count, 8 as expected_count - from {{ ref('DEDUP_CDC_EXCLUDED_SCD') }} -union all - select distinct count(*) as row_count, 4 as expected_count - from {{ ref('DEDUP_CDC_EXCLUDED') }} - -union all - - select distinct count(*) as row_count, 8 as expected_count - from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_POS_DEDUP_CDCX') }} -union all - select distinct count(*) as row_count, 8 as expected_count - from {{ ref('POS_DEDUP_CDCX_SCD') }} -union all - select distinct count(*) as row_count, 3 as expected_count - from {{ ref('POS_DEDUP_CDCX') }} -) -select * -from table_row_counts -where row_count != expected_count diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql index f4ad52f55e61..d1960bc9d6f6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql @@ -1,13 +1,21 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, {{ json_extract_scalar('_airbyte_data', ['name'], ['name']) }} as NAME, - {{ json_extract_scalar('_airbyte_data', ['column`_\'with"_quotes'], ['column___with__quotes']) }} as {{ adapter.quote('column`_\'with""_quotes') }}, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_lsn'], ['_ab_cdc_lsn']) }} as _AB_CDC_LSN, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _AB_CDC_UPDATED_AT, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _AB_CDC_DELETED_AT, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_CDC_EXCLUDED') }} as table_alias -- DEDUP_CDC_EXCLUDED +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql index 39a6396a3344..53573e0731e8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql @@ -1,13 +1,21 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(ID as {{ dbt_utils.type_bigint() }}) as ID, cast(NAME as {{ dbt_utils.type_string() }}) as NAME, - cast({{ adapter.quote('column`_\'with""_quotes') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('column`_\'with""_quotes') }}, cast(_AB_CDC_LSN as {{ dbt_utils.type_float() }}) as _AB_CDC_LSN, cast(_AB_CDC_UPDATED_AT as {{ dbt_utils.type_float() }}) as _AB_CDC_UPDATED_AT, cast(_AB_CDC_DELETED_AT as {{ dbt_utils.type_float() }}) as _AB_CDC_DELETED_AT, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('DEDUP_CDC_EXCLUDED_AB1') }} -- DEDUP_CDC_EXCLUDED +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql deleted file mode 100644 index 44185194f64f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'ID', - 'NAME', - adapter.quote('column`_\'with""_quotes'), - '_AB_CDC_LSN', - '_AB_CDC_UPDATED_AT', - '_AB_CDC_DELETED_AT', - ]) }} as _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID, - tmp.* -from {{ ref('DEDUP_CDC_EXCLUDED_AB2') }} tmp --- DEDUP_CDC_EXCLUDED - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB4.sql deleted file mode 100644 index fa5fa68a458c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID - order by _AIRBYTE_EMITTED_AT asc - ) as _AIRBYTE_ROW_NUM, - tmp.* -from {{ ref('DEDUP_CDC_EXCLUDED_AB3') }} tmp --- DEDUP_CDC_EXCLUDED from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_CDC_EXCLUDED') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql index bdc1a517763e..52686abcae76 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, @@ -9,7 +14,11 @@ select {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as HKD_SPECIAL___CHARACTERS, {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as NZD, {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as USD, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_EXCHANGE_RATE') }} as table_alias -- DEDUP_EXCHANGE_RATE +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql index 1851076c48c9..66aad2b9bd12 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(ID as {{ dbt_utils.type_bigint() }}) as ID, @@ -17,7 +22,11 @@ select cast(HKD_SPECIAL___CHARACTERS as {{ dbt_utils.type_string() }}) as HKD_SPECIAL___CHARACTERS, cast(NZD as {{ dbt_utils.type_float() }}) as NZD, cast(USD as {{ dbt_utils.type_float() }}) as USD, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('DEDUP_EXCHANGE_RATE_AB1') }} -- DEDUP_EXCHANGE_RATE +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql deleted file mode 100644 index 38db984c8b69..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'ID', - 'CURRENCY', - 'DATE', - 'TIMESTAMP_COL', - adapter.quote('HKD@spéçiäl & characters'), - 'HKD_SPECIAL___CHARACTERS', - 'NZD', - 'USD', - ]) }} as _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID, - tmp.* -from {{ ref('DEDUP_EXCHANGE_RATE_AB2') }} tmp --- DEDUP_EXCHANGE_RATE - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB4.sql deleted file mode 100644 index 6cae7e097fe8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID - order by _AIRBYTE_EMITTED_AT asc - ) as _AIRBYTE_ROW_NUM, - tmp.* -from {{ ref('DEDUP_EXCHANGE_RATE_AB3') }} tmp --- DEDUP_EXCHANGE_RATE from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_EXCHANGE_RATE') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql index fffa51f8b038..d2d59c06a80b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, @@ -9,7 +14,11 @@ select {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as HKD_SPECIAL___CHARACTERS, {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as NZD, {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as USD, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_EXCHANGE_RATE') }} as table_alias -- EXCHANGE_RATE +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql index 473aeec716da..d61195d2eb84 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(ID as {{ dbt_utils.type_bigint() }}) as ID, @@ -17,7 +22,11 @@ select cast(HKD_SPECIAL___CHARACTERS as {{ dbt_utils.type_string() }}) as HKD_SPECIAL___CHARACTERS, cast(NZD as {{ dbt_utils.type_float() }}) as NZD, cast(USD as {{ dbt_utils.type_float() }}) as USD, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('EXCHANGE_RATE_AB1') }} -- EXCHANGE_RATE +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql index 239c8e6caf12..db5f1117000f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -14,4 +19,6 @@ select tmp.* from {{ ref('EXCHANGE_RATE_AB2') }} tmp -- EXCHANGE_RATE +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql index b26e74409945..cae6bceaf73d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, @@ -7,7 +12,10 @@ select {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _AB_CDC_UPDATED_AT, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _AB_CDC_DELETED_AT, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_log_pos'], ['_ab_cdc_log_pos']) }} as _AB_CDC_LOG_POS, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_POS_DEDUP_CDCX') }} as table_alias -- POS_DEDUP_CDCX +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql index 361817894307..9e48e834cd99 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(ID as {{ dbt_utils.type_bigint() }}) as ID, @@ -7,7 +12,10 @@ select cast(_AB_CDC_UPDATED_AT as {{ dbt_utils.type_float() }}) as _AB_CDC_UPDATED_AT, cast(_AB_CDC_DELETED_AT as {{ dbt_utils.type_float() }}) as _AB_CDC_DELETED_AT, cast(_AB_CDC_LOG_POS as {{ dbt_utils.type_float() }}) as _AB_CDC_LOG_POS, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('POS_DEDUP_CDCX_AB1') }} -- POS_DEDUP_CDCX +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql deleted file mode 100644 index 743b21efdd28..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'ID', - 'NAME', - '_AB_CDC_LSN', - '_AB_CDC_UPDATED_AT', - '_AB_CDC_DELETED_AT', - '_AB_CDC_LOG_POS', - ]) }} as _AIRBYTE_POS_DEDUP_CDCX_HASHID, - tmp.* -from {{ ref('POS_DEDUP_CDCX_AB2') }} tmp --- POS_DEDUP_CDCX - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB4.sql deleted file mode 100644 index 7e454448034e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _AIRBYTE_POS_DEDUP_CDCX_HASHID - order by _AIRBYTE_EMITTED_AT asc - ) as _AIRBYTE_ROW_NUM, - tmp.* -from {{ ref('POS_DEDUP_CDCX_AB3') }} tmp --- POS_DEDUP_CDCX from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_POS_DEDUP_CDCX') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql new file mode 100644 index 000000000000..de7672d5c9df --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql @@ -0,0 +1,17 @@ +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT +from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_RENAMED_DEDUP_CDC_EXCLUDED') }} as table_alias +-- RENAMED_DEDUP_CDC_EXCLUDED +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql new file mode 100644 index 000000000000..8597d9cbf0fa --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql @@ -0,0 +1,17 @@ +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as {{ dbt_utils.type_bigint() }}) as ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT +from {{ ref('RENAMED_DEDUP_CDC_EXCLUDED_AB1') }} +-- RENAMED_DEDUP_CDC_EXCLUDED +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql new file mode 100644 index 000000000000..6aa5c273499d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql @@ -0,0 +1,24 @@ +{{ config( + cluster_by = ["_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_EMITTED_AT"], + unique_key = "_AIRBYTE_UNIQUE_KEY", + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _AIRBYTE_UNIQUE_KEY, + ID, + NAME, + _AB_CDC_LSN, + _AB_CDC_UPDATED_AT, + _AB_CDC_DELETED_AT, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID +from {{ ref('DEDUP_CDC_EXCLUDED_SCD') }} +-- DEDUP_CDC_EXCLUDED from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_CDC_EXCLUDED') }} +where 1 = 1 +and _AIRBYTE_ACTIVE_ROW = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql new file mode 100644 index 000000000000..e835ffe0c1bb --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql @@ -0,0 +1,27 @@ +{{ config( + cluster_by = ["_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_EMITTED_AT"], + unique_key = "_AIRBYTE_UNIQUE_KEY", + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _AIRBYTE_UNIQUE_KEY, + ID, + CURRENCY, + DATE, + TIMESTAMP_COL, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + HKD_SPECIAL___CHARACTERS, + NZD, + USD, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID +from {{ ref('DEDUP_EXCHANGE_RATE_SCD') }} +-- DEDUP_EXCHANGE_RATE from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_EXCHANGE_RATE') }} +where 1 = 1 +and _AIRBYTE_ACTIVE_ROW = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/EXCHANGE_RATE.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/EXCHANGE_RATE.sql new file mode 100644 index 000000000000..533791bfb80e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/EXCHANGE_RATE.sql @@ -0,0 +1,25 @@ +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + ID, + CURRENCY, + DATE, + TIMESTAMP_COL, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + HKD_SPECIAL___CHARACTERS, + NZD, + USD, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_EXCHANGE_RATE_HASHID +from {{ ref('EXCHANGE_RATE_AB3') }} +-- EXCHANGE_RATE from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_EXCHANGE_RATE') }} +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql new file mode 100644 index 000000000000..6c761d93c54f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql @@ -0,0 +1,20 @@ +{{ config( + cluster_by = ["_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_EMITTED_AT"], + unique_key = "_AIRBYTE_UNIQUE_KEY", + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _AIRBYTE_UNIQUE_KEY, + ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID +from {{ ref('RENAMED_DEDUP_CDC_EXCLUDED_SCD') }} +-- RENAMED_DEDUP_CDC_EXCLUDED from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_RENAMED_DEDUP_CDC_EXCLUDED') }} +where 1 = 1 +and _AIRBYTE_ACTIVE_ROW = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql new file mode 100644 index 000000000000..6fcbf7b8e63d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql @@ -0,0 +1,112 @@ +{{ config( + cluster_by = ["_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_EMITTED_AT"], + unique_key = "_AIRBYTE_UNIQUE_KEY_SCD", + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('DEDUP_CDC_EXCLUDED_AB3') }} + -- DEDUP_CDC_EXCLUDED from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_CDC_EXCLUDED') }} + where 1 = 1 + {{ incremental_clause('_AIRBYTE_EMITTED_AT') }} +), +new_data_ids as ( + -- build a subset of _AIRBYTE_UNIQUE_KEY from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'ID', + ]) }} as _AIRBYTE_UNIQUE_KEY + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('DEDUP_CDC_EXCLUDED_AB3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._AIRBYTE_UNIQUE_KEY = new_data_ids._AIRBYTE_UNIQUE_KEY + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('DEDUP_CDC_EXCLUDED_AB3') }} as inc_data on 1 = 0 + where _AIRBYTE_ACTIVE_ROW = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('DEDUP_CDC_EXCLUDED_AB3')) }} from new_data + union all + select {{ dbt_utils.star(ref('DEDUP_CDC_EXCLUDED_AB3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('DEDUP_CDC_EXCLUDED_AB3') }} + -- DEDUP_CDC_EXCLUDED from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_CDC_EXCLUDED') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'ID', + ]) }} as _AIRBYTE_UNIQUE_KEY, + ID, + NAME, + _AB_CDC_LSN, + _AB_CDC_UPDATED_AT, + _AB_CDC_DELETED_AT, + _AIRBYTE_EMITTED_AT as _AIRBYTE_START_AT, + lag(_AIRBYTE_EMITTED_AT) over ( + partition by ID + order by + _AIRBYTE_EMITTED_AT is null asc, + _AIRBYTE_EMITTED_AT desc, + _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc + ) as _AIRBYTE_END_AT, + case when lag(_AIRBYTE_EMITTED_AT) over ( + partition by ID + order by + _AIRBYTE_EMITTED_AT is null asc, + _AIRBYTE_EMITTED_AT desc, + _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc + ) is null and _AB_CDC_DELETED_AT is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT, cast(_AB_CDC_DELETED_AT as {{ dbt_utils.type_string() }}), cast(_AB_CDC_UPDATED_AT as {{ dbt_utils.type_string() }}) + order by _AIRBYTE_AB_ID + ) as _AIRBYTE_ROW_NUM, + {{ dbt_utils.surrogate_key([ + '_AIRBYTE_UNIQUE_KEY', + '_AIRBYTE_START_AT', + '_AIRBYTE_EMITTED_AT', '_AB_CDC_DELETED_AT', '_AB_CDC_UPDATED_AT' + ]) }} as _AIRBYTE_UNIQUE_KEY_SCD, + scd_data.* + from scd_data +) +select + _AIRBYTE_UNIQUE_KEY, + _AIRBYTE_UNIQUE_KEY_SCD, + ID, + NAME, + _AB_CDC_LSN, + _AB_CDC_UPDATED_AT, + _AB_CDC_DELETED_AT, + _AIRBYTE_START_AT, + _AIRBYTE_END_AT, + _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID +from dedup_data where _AIRBYTE_ROW_NUM = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql new file mode 100644 index 000000000000..060a3ccb799a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql @@ -0,0 +1,122 @@ +{{ config( + cluster_by = ["_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_EMITTED_AT"], + unique_key = "_AIRBYTE_UNIQUE_KEY_SCD", + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('DEDUP_EXCHANGE_RATE_AB3') }} + -- DEDUP_EXCHANGE_RATE from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_EXCHANGE_RATE') }} + where 1 = 1 + {{ incremental_clause('_AIRBYTE_EMITTED_AT') }} +), +new_data_ids as ( + -- build a subset of _AIRBYTE_UNIQUE_KEY from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'ID', + 'CURRENCY', + 'NZD', + ]) }} as _AIRBYTE_UNIQUE_KEY + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('DEDUP_EXCHANGE_RATE_AB3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._AIRBYTE_UNIQUE_KEY = new_data_ids._AIRBYTE_UNIQUE_KEY + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('DEDUP_EXCHANGE_RATE_AB3') }} as inc_data on 1 = 0 + where _AIRBYTE_ACTIVE_ROW = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('DEDUP_EXCHANGE_RATE_AB3')) }} from new_data + union all + select {{ dbt_utils.star(ref('DEDUP_EXCHANGE_RATE_AB3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('DEDUP_EXCHANGE_RATE_AB3') }} + -- DEDUP_EXCHANGE_RATE from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_EXCHANGE_RATE') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'ID', + 'CURRENCY', + 'NZD', + ]) }} as _AIRBYTE_UNIQUE_KEY, + ID, + CURRENCY, + DATE, + TIMESTAMP_COL, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + HKD_SPECIAL___CHARACTERS, + NZD, + USD, + DATE as _AIRBYTE_START_AT, + lag(DATE) over ( + partition by ID, CURRENCY, cast(NZD as {{ dbt_utils.type_string() }}) + order by + DATE is null asc, + DATE desc, + _AIRBYTE_EMITTED_AT desc + ) as _AIRBYTE_END_AT, + case when lag(DATE) over ( + partition by ID, CURRENCY, cast(NZD as {{ dbt_utils.type_string() }}) + order by + DATE is null asc, + DATE desc, + _AIRBYTE_EMITTED_AT desc + ) is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT + order by _AIRBYTE_AB_ID + ) as _AIRBYTE_ROW_NUM, + {{ dbt_utils.surrogate_key([ + '_AIRBYTE_UNIQUE_KEY', + '_AIRBYTE_START_AT', + '_AIRBYTE_EMITTED_AT' + ]) }} as _AIRBYTE_UNIQUE_KEY_SCD, + scd_data.* + from scd_data +) +select + _AIRBYTE_UNIQUE_KEY, + _AIRBYTE_UNIQUE_KEY_SCD, + ID, + CURRENCY, + DATE, + TIMESTAMP_COL, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + HKD_SPECIAL___CHARACTERS, + NZD, + USD, + _AIRBYTE_START_AT, + _AIRBYTE_END_AT, + _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID +from dedup_data where _AIRBYTE_ROW_NUM = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql new file mode 100644 index 000000000000..c212769bc05f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql @@ -0,0 +1,104 @@ +{{ config( + cluster_by = ["_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_EMITTED_AT"], + unique_key = "_AIRBYTE_UNIQUE_KEY_SCD", + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('RENAMED_DEDUP_CDC_EXCLUDED_AB3') }} + -- RENAMED_DEDUP_CDC_EXCLUDED from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_RENAMED_DEDUP_CDC_EXCLUDED') }} + where 1 = 1 + {{ incremental_clause('_AIRBYTE_EMITTED_AT') }} +), +new_data_ids as ( + -- build a subset of _AIRBYTE_UNIQUE_KEY from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'ID', + ]) }} as _AIRBYTE_UNIQUE_KEY + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('RENAMED_DEDUP_CDC_EXCLUDED_AB3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._AIRBYTE_UNIQUE_KEY = new_data_ids._AIRBYTE_UNIQUE_KEY + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('RENAMED_DEDUP_CDC_EXCLUDED_AB3') }} as inc_data on 1 = 0 + where _AIRBYTE_ACTIVE_ROW = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('RENAMED_DEDUP_CDC_EXCLUDED_AB3')) }} from new_data + union all + select {{ dbt_utils.star(ref('RENAMED_DEDUP_CDC_EXCLUDED_AB3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('RENAMED_DEDUP_CDC_EXCLUDED_AB3') }} + -- RENAMED_DEDUP_CDC_EXCLUDED from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_RENAMED_DEDUP_CDC_EXCLUDED') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'ID', + ]) }} as _AIRBYTE_UNIQUE_KEY, + ID, + _AIRBYTE_EMITTED_AT as _AIRBYTE_START_AT, + lag(_AIRBYTE_EMITTED_AT) over ( + partition by ID + order by + _AIRBYTE_EMITTED_AT is null asc, + _AIRBYTE_EMITTED_AT desc, + _AIRBYTE_EMITTED_AT desc + ) as _AIRBYTE_END_AT, + case when lag(_AIRBYTE_EMITTED_AT) over ( + partition by ID + order by + _AIRBYTE_EMITTED_AT is null asc, + _AIRBYTE_EMITTED_AT desc, + _AIRBYTE_EMITTED_AT desc + ) is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + _AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT + order by _AIRBYTE_AB_ID + ) as _AIRBYTE_ROW_NUM, + {{ dbt_utils.surrogate_key([ + '_AIRBYTE_UNIQUE_KEY', + '_AIRBYTE_START_AT', + '_AIRBYTE_EMITTED_AT' + ]) }} as _AIRBYTE_UNIQUE_KEY_SCD, + scd_data.* + from scd_data +) +select + _AIRBYTE_UNIQUE_KEY, + _AIRBYTE_UNIQUE_KEY_SCD, + ID, + _AIRBYTE_START_AT, + _AIRBYTE_END_AT, + _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID +from dedup_data where _AIRBYTE_ROW_NUM = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql deleted file mode 100644 index 8335a889e120..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["top-level"]) }} --- Final base SQL model -select - ID, - NAME, - {{ adapter.quote('column`_\'with""_quotes') }}, - _AB_CDC_LSN, - _AB_CDC_UPDATED_AT, - _AB_CDC_DELETED_AT, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID -from {{ ref('DEDUP_CDC_EXCLUDED_SCD') }} --- DEDUP_CDC_EXCLUDED from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_CDC_EXCLUDED') }} -where _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql deleted file mode 100644 index bc77ae8e0646..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - ID, - NAME, - {{ adapter.quote('column`_\'with""_quotes') }}, - _AB_CDC_LSN, - _AB_CDC_UPDATED_AT, - _AB_CDC_DELETED_AT, - _AIRBYTE_EMITTED_AT as _AIRBYTE_START_AT, - lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by _AIRBYTE_EMITTED_AT is null asc, _AIRBYTE_EMITTED_AT desc, _AIRBYTE_EMITTED_AT desc - ) as _AIRBYTE_END_AT, - case when lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by _AIRBYTE_EMITTED_AT is null asc, _AIRBYTE_EMITTED_AT desc, _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc - ) is null and _AB_CDC_DELETED_AT is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID -from {{ ref('DEDUP_CDC_EXCLUDED_AB4') }} --- DEDUP_CDC_EXCLUDED from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_CDC_EXCLUDED') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql deleted file mode 100644 index 531a13cf7b5a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["top-level"]) }} --- Final base SQL model -select - ID, - CURRENCY, - DATE, - TIMESTAMP_COL, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - HKD_SPECIAL___CHARACTERS, - NZD, - USD, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID -from {{ ref('DEDUP_EXCHANGE_RATE_SCD') }} --- DEDUP_EXCHANGE_RATE from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_EXCHANGE_RATE') }} -where _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql deleted file mode 100644 index f5f0c288d57f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql +++ /dev/null @@ -1,26 +0,0 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - ID, - CURRENCY, - DATE, - TIMESTAMP_COL, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - HKD_SPECIAL___CHARACTERS, - NZD, - USD, - DATE as _AIRBYTE_START_AT, - lag(DATE) over ( - partition by ID, CURRENCY, cast(NZD as {{ dbt_utils.type_string() }}) - order by DATE is null asc, DATE desc, _AIRBYTE_EMITTED_AT desc - ) as _AIRBYTE_END_AT, - case when lag(DATE) over ( - partition by ID, CURRENCY, cast(NZD as {{ dbt_utils.type_string() }}) - order by DATE is null asc, DATE desc, _AIRBYTE_EMITTED_AT desc - ) is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID -from {{ ref('DEDUP_EXCHANGE_RATE_AB4') }} --- DEDUP_EXCHANGE_RATE from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_EXCHANGE_RATE') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql deleted file mode 100644 index cfaae19243b7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["top-level"]) }} --- Final base SQL model -select - ID, - CURRENCY, - DATE, - TIMESTAMP_COL, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - HKD_SPECIAL___CHARACTERS, - NZD, - USD, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_EXCHANGE_RATE_HASHID -from {{ ref('EXCHANGE_RATE_AB3') }} --- EXCHANGE_RATE from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_EXCHANGE_RATE') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql index 27120ccce874..1ab6634ee9b5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql @@ -1,15 +1,24 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["top-level"]) }} +{{ config( + cluster_by = ["_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_EMITTED_AT"], + unique_key = "_AIRBYTE_UNIQUE_KEY", + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} -- Final base SQL model select + _AIRBYTE_UNIQUE_KEY, ID, NAME, _AB_CDC_LSN, _AB_CDC_UPDATED_AT, _AB_CDC_DELETED_AT, _AB_CDC_LOG_POS, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_POS_DEDUP_CDCX_HASHID from {{ ref('POS_DEDUP_CDCX_SCD') }} -- POS_DEDUP_CDCX from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_POS_DEDUP_CDCX') }} -where _airbyte_active_row = 1 +where 1 = 1 +and _AIRBYTE_ACTIVE_ROW = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql deleted file mode 100644 index 0ab8c0427373..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - ID, - NAME, - _AB_CDC_LSN, - _AB_CDC_UPDATED_AT, - _AB_CDC_DELETED_AT, - _AB_CDC_LOG_POS, - _AIRBYTE_EMITTED_AT as _AIRBYTE_START_AT, - lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by _AIRBYTE_EMITTED_AT is null asc, _AIRBYTE_EMITTED_AT desc, _AIRBYTE_EMITTED_AT desc - ) as _AIRBYTE_END_AT, - case when lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by _AIRBYTE_EMITTED_AT is null asc, _AIRBYTE_EMITTED_AT desc, _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc, _AB_CDC_LOG_POS desc - ) is null and _AB_CDC_DELETED_AT is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_POS_DEDUP_CDCX_HASHID -from {{ ref('POS_DEDUP_CDCX_AB4') }} --- POS_DEDUP_CDCX from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_POS_DEDUP_CDCX') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql new file mode 100644 index 000000000000..257fa09afe15 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql @@ -0,0 +1,114 @@ +{{ config( + cluster_by = ["_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_EMITTED_AT"], + unique_key = "_AIRBYTE_UNIQUE_KEY_SCD", + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('POS_DEDUP_CDCX_AB3') }} + -- POS_DEDUP_CDCX from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_POS_DEDUP_CDCX') }} + where 1 = 1 + {{ incremental_clause('_AIRBYTE_EMITTED_AT') }} +), +new_data_ids as ( + -- build a subset of _AIRBYTE_UNIQUE_KEY from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'ID', + ]) }} as _AIRBYTE_UNIQUE_KEY + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('POS_DEDUP_CDCX_AB3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._AIRBYTE_UNIQUE_KEY = new_data_ids._AIRBYTE_UNIQUE_KEY + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('POS_DEDUP_CDCX_AB3') }} as inc_data on 1 = 0 + where _AIRBYTE_ACTIVE_ROW = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('POS_DEDUP_CDCX_AB3')) }} from new_data + union all + select {{ dbt_utils.star(ref('POS_DEDUP_CDCX_AB3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('POS_DEDUP_CDCX_AB3') }} + -- POS_DEDUP_CDCX from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_POS_DEDUP_CDCX') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'ID', + ]) }} as _AIRBYTE_UNIQUE_KEY, + ID, + NAME, + _AB_CDC_LSN, + _AB_CDC_UPDATED_AT, + _AB_CDC_DELETED_AT, + _AB_CDC_LOG_POS, + _AIRBYTE_EMITTED_AT as _AIRBYTE_START_AT, + lag(_AIRBYTE_EMITTED_AT) over ( + partition by ID + order by + _AIRBYTE_EMITTED_AT is null asc, + _AIRBYTE_EMITTED_AT desc, + _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc, _AB_CDC_LOG_POS desc + ) as _AIRBYTE_END_AT, + case when lag(_AIRBYTE_EMITTED_AT) over ( + partition by ID + order by + _AIRBYTE_EMITTED_AT is null asc, + _AIRBYTE_EMITTED_AT desc, + _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc, _AB_CDC_LOG_POS desc + ) is null and _AB_CDC_DELETED_AT is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + _AIRBYTE_POS_DEDUP_CDCX_HASHID + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT, cast(_AB_CDC_DELETED_AT as {{ dbt_utils.type_string() }}), cast(_AB_CDC_UPDATED_AT as {{ dbt_utils.type_string() }}), cast(_AB_CDC_LOG_POS as {{ dbt_utils.type_string() }}) + order by _AIRBYTE_AB_ID + ) as _AIRBYTE_ROW_NUM, + {{ dbt_utils.surrogate_key([ + '_AIRBYTE_UNIQUE_KEY', + '_AIRBYTE_START_AT', + '_AIRBYTE_EMITTED_AT', '_AB_CDC_DELETED_AT', '_AB_CDC_UPDATED_AT', '_AB_CDC_LOG_POS' + ]) }} as _AIRBYTE_UNIQUE_KEY_SCD, + scd_data.* + from scd_data +) +select + _AIRBYTE_UNIQUE_KEY, + _AIRBYTE_UNIQUE_KEY_SCD, + ID, + NAME, + _AB_CDC_LSN, + _AB_CDC_UPDATED_AT, + _AB_CDC_DELETED_AT, + _AB_CDC_LOG_POS, + _AIRBYTE_START_AT, + _AIRBYTE_END_AT, + _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_POS_DEDUP_CDCX_HASHID +from dedup_data where _AIRBYTE_ROW_NUM = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql new file mode 100644 index 000000000000..1899392af6fc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql @@ -0,0 +1,21 @@ +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'ID', + 'NAME', + '_AB_CDC_LSN', + '_AB_CDC_UPDATED_AT', + '_AB_CDC_DELETED_AT', + ]) }} as _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID, + tmp.* +from {{ ref('DEDUP_CDC_EXCLUDED_AB2') }} tmp +-- DEDUP_CDC_EXCLUDED +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql new file mode 100644 index 000000000000..a78412000006 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql @@ -0,0 +1,24 @@ +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'ID', + 'CURRENCY', + 'DATE', + 'TIMESTAMP_COL', + adapter.quote('HKD@spéçiäl & characters'), + 'HKD_SPECIAL___CHARACTERS', + 'NZD', + 'USD', + ]) }} as _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID, + tmp.* +from {{ ref('DEDUP_EXCHANGE_RATE_AB2') }} tmp +-- DEDUP_EXCHANGE_RATE +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql new file mode 100644 index 000000000000..c886fcf1eadf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql @@ -0,0 +1,21 @@ +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'ID', + 'NAME', + '_AB_CDC_LSN', + '_AB_CDC_UPDATED_AT', + '_AB_CDC_DELETED_AT', + '_AB_CDC_LOG_POS', + ]) }} as _AIRBYTE_POS_DEDUP_CDCX_HASHID, + tmp.* +from {{ ref('POS_DEDUP_CDCX_AB2') }} tmp +-- POS_DEDUP_CDCX +where 1 = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql new file mode 100644 index 000000000000..d3b3c50580d0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql @@ -0,0 +1,17 @@ +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'ID', + ]) }} as _AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID, + tmp.* +from {{ ref('RENAMED_DEDUP_CDC_EXCLUDED_AB2') }} tmp +-- RENAMED_DEDUP_CDC_EXCLUDED +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/sources.yml index a8123e03f06b..c09293ca6726 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/sources.yml @@ -10,3 +10,4 @@ sources: - name: _AIRBYTE_RAW_DEDUP_EXCHANGE_RATE - name: _AIRBYTE_RAW_EXCHANGE_RATE - name: _AIRBYTE_RAW_POS_DEDUP_CDCX + - name: _AIRBYTE_RAW_RENAMED_DEDUP_CDC_EXCLUDED diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql new file mode 100644 index 000000000000..8d5282887fb0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql @@ -0,0 +1,18 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"name"')) as NAME, + to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_lsn"')) as _AB_CDC_LSN, + to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_updated_at"')) as _AB_CDC_UPDATED_AT, + to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_deleted_at"')) as _AB_CDC_DELETED_AT, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_CDC_EXCLUDED as table_alias +-- DEDUP_CDC_EXCLUDED +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql new file mode 100644 index 000000000000..57a50d43e5bc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql @@ -0,0 +1,28 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + bigint +) as ID, + cast(NAME as + varchar +) as NAME, + cast(_AB_CDC_LSN as + float +) as _AB_CDC_LSN, + cast(_AB_CDC_UPDATED_AT as + float +) as _AB_CDC_UPDATED_AT, + cast(_AB_CDC_DELETED_AT as + float +) as _AB_CDC_DELETED_AT, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB1" +-- DEDUP_CDC_EXCLUDED +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql new file mode 100644 index 000000000000..ee207b768a20 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql @@ -0,0 +1,21 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"currency"')) as CURRENCY, + to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, + to_varchar(get_path(parse_json(_airbyte_data), '"timestamp_col"')) as TIMESTAMP_COL, + to_varchar(get_path(parse_json(_airbyte_data), '"HKD@spéçiäl & characters"')) as "HKD@spéçiäl & characters", + to_varchar(get_path(parse_json(_airbyte_data), '"HKD_special___characters"')) as HKD_SPECIAL___CHARACTERS, + to_varchar(get_path(parse_json(_airbyte_data), '"NZD"')) as NZD, + to_varchar(get_path(parse_json(_airbyte_data), '"USD"')) as USD, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_EXCHANGE_RATE as table_alias +-- DEDUP_EXCHANGE_RATE +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql new file mode 100644 index 000000000000..66b09ec2508e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql @@ -0,0 +1,43 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + bigint +) as ID, + cast(CURRENCY as + varchar +) as CURRENCY, + cast(nullif(DATE, '') as + date +) as DATE, + case + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZHTZM') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZH') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZHTZM') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZH') + when TIMESTAMP_COL = '' then NULL + else to_timestamp_tz(TIMESTAMP_COL) + end as TIMESTAMP_COL + , + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(HKD_SPECIAL___CHARACTERS as + varchar +) as HKD_SPECIAL___CHARACTERS, + cast(NZD as + float +) as NZD, + cast(USD as + float +) as USD, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB1" +-- DEDUP_EXCHANGE_RATE +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql new file mode 100644 index 000000000000..33e07a844a1c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql @@ -0,0 +1,21 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"currency"')) as CURRENCY, + to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, + to_varchar(get_path(parse_json(_airbyte_data), '"timestamp_col"')) as TIMESTAMP_COL, + to_varchar(get_path(parse_json(_airbyte_data), '"HKD@spéçiäl & characters"')) as "HKD@spéçiäl & characters", + to_varchar(get_path(parse_json(_airbyte_data), '"HKD_special___characters"')) as HKD_SPECIAL___CHARACTERS, + to_varchar(get_path(parse_json(_airbyte_data), '"NZD"')) as NZD, + to_varchar(get_path(parse_json(_airbyte_data), '"USD"')) as USD, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_EXCHANGE_RATE as table_alias +-- EXCHANGE_RATE +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql new file mode 100644 index 000000000000..f5ef6e473984 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql @@ -0,0 +1,43 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + bigint +) as ID, + cast(CURRENCY as + varchar +) as CURRENCY, + cast(nullif(DATE, '') as + date +) as DATE, + case + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZHTZM') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZH') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZHTZM') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZH') + when TIMESTAMP_COL = '' then NULL + else to_timestamp_tz(TIMESTAMP_COL) + end as TIMESTAMP_COL + , + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(HKD_SPECIAL___CHARACTERS as + varchar +) as HKD_SPECIAL___CHARACTERS, + cast(NZD as + float +) as NZD, + cast(USD as + float +) as USD, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB1" +-- EXCHANGE_RATE +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql new file mode 100644 index 000000000000..c1f0840c3677 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql @@ -0,0 +1,30 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(CURRENCY as + varchar +), '') || '-' || coalesce(cast(DATE as + varchar +), '') || '-' || coalesce(cast(TIMESTAMP_COL as + varchar +), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as + varchar +), '') || '-' || coalesce(cast(HKD_SPECIAL___CHARACTERS as + varchar +), '') || '-' || coalesce(cast(NZD as + varchar +), '') || '-' || coalesce(cast(USD as + varchar +), '') as + varchar +)) as _AIRBYTE_EXCHANGE_RATE_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB2" tmp +-- EXCHANGE_RATE +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql new file mode 100644 index 000000000000..93d03ade5063 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql @@ -0,0 +1,18 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"name"')) as NAME, + to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_lsn"')) as _AB_CDC_LSN, + to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_updated_at"')) as _AB_CDC_UPDATED_AT, + to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_deleted_at"')) as _AB_CDC_DELETED_AT, + to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_log_pos"')) as _AB_CDC_LOG_POS, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_POS_DEDUP_CDCX as table_alias +-- POS_DEDUP_CDCX +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql new file mode 100644 index 000000000000..82b6ca1c2619 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql @@ -0,0 +1,30 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + bigint +) as ID, + cast(NAME as + varchar +) as NAME, + cast(_AB_CDC_LSN as + float +) as _AB_CDC_LSN, + cast(_AB_CDC_UPDATED_AT as + float +) as _AB_CDC_UPDATED_AT, + cast(_AB_CDC_DELETED_AT as + float +) as _AB_CDC_DELETED_AT, + cast(_AB_CDC_LOG_POS as + float +) as _AB_CDC_LOG_POS, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB1" +-- POS_DEDUP_CDCX +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql new file mode 100644 index 000000000000..123971c4e5dc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql @@ -0,0 +1,14 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_RENAMED_DEDUP_CDC_EXCLUDED as table_alias +-- RENAMED_DEDUP_CDC_EXCLUDED +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql new file mode 100644 index 000000000000..35438f37c247 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + bigint +) as ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB1" +-- RENAMED_DEDUP_CDC_EXCLUDED +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql new file mode 100644 index 000000000000..522240329e42 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql @@ -0,0 +1,26 @@ +begin; + + + + + + + + merge into "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED" as DBT_INTERNAL_DEST + using "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED__dbt_tmp" as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._AIRBYTE_UNIQUE_KEY = DBT_INTERNAL_DEST._AIRBYTE_UNIQUE_KEY + + + + when matched then update set + "_AIRBYTE_UNIQUE_KEY" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY","ID" = DBT_INTERNAL_SOURCE."ID","NAME" = DBT_INTERNAL_SOURCE."NAME","_AB_CDC_LSN" = DBT_INTERNAL_SOURCE."_AB_CDC_LSN","_AB_CDC_UPDATED_AT" = DBT_INTERNAL_SOURCE."_AB_CDC_UPDATED_AT","_AB_CDC_DELETED_AT" = DBT_INTERNAL_SOURCE."_AB_CDC_DELETED_AT","_AIRBYTE_AB_ID" = DBT_INTERNAL_SOURCE."_AIRBYTE_AB_ID","_AIRBYTE_EMITTED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_EMITTED_AT","_AIRBYTE_NORMALIZED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_NORMALIZED_AT","_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" = DBT_INTERNAL_SOURCE."_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" + + + when not matched then insert + ("_AIRBYTE_UNIQUE_KEY", "ID", "NAME", "_AB_CDC_LSN", "_AB_CDC_UPDATED_AT", "_AB_CDC_DELETED_AT", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID") + values + ("_AIRBYTE_UNIQUE_KEY", "ID", "NAME", "_AB_CDC_LSN", "_AB_CDC_UPDATED_AT", "_AB_CDC_DELETED_AT", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID") + +; + commit; \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql new file mode 100644 index 000000000000..00f8af4d282f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql @@ -0,0 +1,26 @@ +begin; + + + + + + + + merge into "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE" as DBT_INTERNAL_DEST + using "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE__dbt_tmp" as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._AIRBYTE_UNIQUE_KEY = DBT_INTERNAL_DEST._AIRBYTE_UNIQUE_KEY + + + + when matched then update set + "_AIRBYTE_UNIQUE_KEY" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY","ID" = DBT_INTERNAL_SOURCE."ID","CURRENCY" = DBT_INTERNAL_SOURCE."CURRENCY","DATE" = DBT_INTERNAL_SOURCE."DATE","TIMESTAMP_COL" = DBT_INTERNAL_SOURCE."TIMESTAMP_COL","HKD@spéçiäl & characters" = DBT_INTERNAL_SOURCE."HKD@spéçiäl & characters","HKD_SPECIAL___CHARACTERS" = DBT_INTERNAL_SOURCE."HKD_SPECIAL___CHARACTERS","NZD" = DBT_INTERNAL_SOURCE."NZD","USD" = DBT_INTERNAL_SOURCE."USD","_AIRBYTE_AB_ID" = DBT_INTERNAL_SOURCE."_AIRBYTE_AB_ID","_AIRBYTE_EMITTED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_EMITTED_AT","_AIRBYTE_NORMALIZED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_NORMALIZED_AT","_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID" = DBT_INTERNAL_SOURCE."_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID" + + + when not matched then insert + ("_AIRBYTE_UNIQUE_KEY", "ID", "CURRENCY", "DATE", "TIMESTAMP_COL", "HKD@spéçiäl & characters", "HKD_SPECIAL___CHARACTERS", "NZD", "USD", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID") + values + ("_AIRBYTE_UNIQUE_KEY", "ID", "CURRENCY", "DATE", "TIMESTAMP_COL", "HKD@spéçiäl & characters", "HKD_SPECIAL___CHARACTERS", "NZD", "USD", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID") + +; + commit; \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/EXCHANGE_RATE.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/EXCHANGE_RATE.sql new file mode 100644 index 000000000000..fc67dd05b1bf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/EXCHANGE_RATE.sql @@ -0,0 +1,26 @@ +begin; + + + + + + + + merge into "AIRBYTE_DATABASE".TEST_NORMALIZATION."EXCHANGE_RATE" as DBT_INTERNAL_DEST + using "AIRBYTE_DATABASE".TEST_NORMALIZATION."EXCHANGE_RATE__dbt_tmp" as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._AIRBYTE_AB_ID = DBT_INTERNAL_DEST._AIRBYTE_AB_ID + + + + when matched then update set + "ID" = DBT_INTERNAL_SOURCE."ID","CURRENCY" = DBT_INTERNAL_SOURCE."CURRENCY","DATE" = DBT_INTERNAL_SOURCE."DATE","TIMESTAMP_COL" = DBT_INTERNAL_SOURCE."TIMESTAMP_COL","HKD@spéçiäl & characters" = DBT_INTERNAL_SOURCE."HKD@spéçiäl & characters","HKD_SPECIAL___CHARACTERS" = DBT_INTERNAL_SOURCE."HKD_SPECIAL___CHARACTERS","NZD" = DBT_INTERNAL_SOURCE."NZD","USD" = DBT_INTERNAL_SOURCE."USD","_AIRBYTE_AB_ID" = DBT_INTERNAL_SOURCE."_AIRBYTE_AB_ID","_AIRBYTE_EMITTED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_EMITTED_AT","_AIRBYTE_NORMALIZED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_NORMALIZED_AT","_AIRBYTE_EXCHANGE_RATE_HASHID" = DBT_INTERNAL_SOURCE."_AIRBYTE_EXCHANGE_RATE_HASHID" + + + when not matched then insert + ("ID", "CURRENCY", "DATE", "TIMESTAMP_COL", "HKD@spéçiäl & characters", "HKD_SPECIAL___CHARACTERS", "NZD", "USD", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_EXCHANGE_RATE_HASHID") + values + ("ID", "CURRENCY", "DATE", "TIMESTAMP_COL", "HKD@spéçiäl & characters", "HKD_SPECIAL___CHARACTERS", "NZD", "USD", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_EXCHANGE_RATE_HASHID") + +; + commit; \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql new file mode 100644 index 000000000000..3ddd6bd2a029 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql @@ -0,0 +1,26 @@ +begin; + + + + + + + + merge into "AIRBYTE_DATABASE".TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED" as DBT_INTERNAL_DEST + using "AIRBYTE_DATABASE".TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED__dbt_tmp" as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._AIRBYTE_UNIQUE_KEY = DBT_INTERNAL_DEST._AIRBYTE_UNIQUE_KEY + + + + when matched then update set + "_AIRBYTE_UNIQUE_KEY" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY","ID" = DBT_INTERNAL_SOURCE."ID","_AIRBYTE_AB_ID" = DBT_INTERNAL_SOURCE."_AIRBYTE_AB_ID","_AIRBYTE_EMITTED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_EMITTED_AT","_AIRBYTE_NORMALIZED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_NORMALIZED_AT","_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" = DBT_INTERNAL_SOURCE."_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" + + + when not matched then insert + ("_AIRBYTE_UNIQUE_KEY", "ID", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID") + values + ("_AIRBYTE_UNIQUE_KEY", "ID", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID") + +; + commit; \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql new file mode 100644 index 000000000000..823c37218a10 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql @@ -0,0 +1,26 @@ +begin; + + + + + + + + merge into "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_SCD" as DBT_INTERNAL_DEST + using "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_SCD__dbt_tmp" as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._AIRBYTE_UNIQUE_KEY_SCD = DBT_INTERNAL_DEST._AIRBYTE_UNIQUE_KEY_SCD + + + + when matched then update set + "_AIRBYTE_UNIQUE_KEY" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY","_AIRBYTE_UNIQUE_KEY_SCD" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY_SCD","ID" = DBT_INTERNAL_SOURCE."ID","NAME" = DBT_INTERNAL_SOURCE."NAME","_AB_CDC_LSN" = DBT_INTERNAL_SOURCE."_AB_CDC_LSN","_AB_CDC_UPDATED_AT" = DBT_INTERNAL_SOURCE."_AB_CDC_UPDATED_AT","_AB_CDC_DELETED_AT" = DBT_INTERNAL_SOURCE."_AB_CDC_DELETED_AT","_AIRBYTE_START_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_START_AT","_AIRBYTE_END_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_END_AT","_AIRBYTE_ACTIVE_ROW" = DBT_INTERNAL_SOURCE."_AIRBYTE_ACTIVE_ROW","_AIRBYTE_AB_ID" = DBT_INTERNAL_SOURCE."_AIRBYTE_AB_ID","_AIRBYTE_EMITTED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_EMITTED_AT","_AIRBYTE_NORMALIZED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_NORMALIZED_AT","_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" = DBT_INTERNAL_SOURCE."_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" + + + when not matched then insert + ("_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_UNIQUE_KEY_SCD", "ID", "NAME", "_AB_CDC_LSN", "_AB_CDC_UPDATED_AT", "_AB_CDC_DELETED_AT", "_AIRBYTE_START_AT", "_AIRBYTE_END_AT", "_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID") + values + ("_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_UNIQUE_KEY_SCD", "ID", "NAME", "_AB_CDC_LSN", "_AB_CDC_UPDATED_AT", "_AB_CDC_DELETED_AT", "_AIRBYTE_START_AT", "_AIRBYTE_END_AT", "_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID") + +; + commit; \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql new file mode 100644 index 000000000000..f4b7a7831cdf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql @@ -0,0 +1,26 @@ +begin; + + + + + + + + merge into "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_SCD" as DBT_INTERNAL_DEST + using "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_SCD__dbt_tmp" as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._AIRBYTE_UNIQUE_KEY_SCD = DBT_INTERNAL_DEST._AIRBYTE_UNIQUE_KEY_SCD + + + + when matched then update set + "_AIRBYTE_UNIQUE_KEY" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY","_AIRBYTE_UNIQUE_KEY_SCD" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY_SCD","ID" = DBT_INTERNAL_SOURCE."ID","CURRENCY" = DBT_INTERNAL_SOURCE."CURRENCY","DATE" = DBT_INTERNAL_SOURCE."DATE","TIMESTAMP_COL" = DBT_INTERNAL_SOURCE."TIMESTAMP_COL","HKD@spéçiäl & characters" = DBT_INTERNAL_SOURCE."HKD@spéçiäl & characters","HKD_SPECIAL___CHARACTERS" = DBT_INTERNAL_SOURCE."HKD_SPECIAL___CHARACTERS","NZD" = DBT_INTERNAL_SOURCE."NZD","USD" = DBT_INTERNAL_SOURCE."USD","_AIRBYTE_START_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_START_AT","_AIRBYTE_END_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_END_AT","_AIRBYTE_ACTIVE_ROW" = DBT_INTERNAL_SOURCE."_AIRBYTE_ACTIVE_ROW","_AIRBYTE_AB_ID" = DBT_INTERNAL_SOURCE."_AIRBYTE_AB_ID","_AIRBYTE_EMITTED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_EMITTED_AT","_AIRBYTE_NORMALIZED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_NORMALIZED_AT","_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID" = DBT_INTERNAL_SOURCE."_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID" + + + when not matched then insert + ("_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_UNIQUE_KEY_SCD", "ID", "CURRENCY", "DATE", "TIMESTAMP_COL", "HKD@spéçiäl & characters", "HKD_SPECIAL___CHARACTERS", "NZD", "USD", "_AIRBYTE_START_AT", "_AIRBYTE_END_AT", "_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID") + values + ("_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_UNIQUE_KEY_SCD", "ID", "CURRENCY", "DATE", "TIMESTAMP_COL", "HKD@spéçiäl & characters", "HKD_SPECIAL___CHARACTERS", "NZD", "USD", "_AIRBYTE_START_AT", "_AIRBYTE_END_AT", "_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID") + +; + commit; \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql new file mode 100644 index 000000000000..f95adde306f8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql @@ -0,0 +1,26 @@ +begin; + + + + + + + + merge into "AIRBYTE_DATABASE".TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_SCD" as DBT_INTERNAL_DEST + using "AIRBYTE_DATABASE".TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_SCD__dbt_tmp" as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._AIRBYTE_UNIQUE_KEY_SCD = DBT_INTERNAL_DEST._AIRBYTE_UNIQUE_KEY_SCD + + + + when matched then update set + "_AIRBYTE_UNIQUE_KEY" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY","_AIRBYTE_UNIQUE_KEY_SCD" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY_SCD","ID" = DBT_INTERNAL_SOURCE."ID","_AIRBYTE_START_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_START_AT","_AIRBYTE_END_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_END_AT","_AIRBYTE_ACTIVE_ROW" = DBT_INTERNAL_SOURCE."_AIRBYTE_ACTIVE_ROW","_AIRBYTE_AB_ID" = DBT_INTERNAL_SOURCE."_AIRBYTE_AB_ID","_AIRBYTE_EMITTED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_EMITTED_AT","_AIRBYTE_NORMALIZED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_NORMALIZED_AT","_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" = DBT_INTERNAL_SOURCE."_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" + + + when not matched then insert + ("_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_UNIQUE_KEY_SCD", "ID", "_AIRBYTE_START_AT", "_AIRBYTE_END_AT", "_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID") + values + ("_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_UNIQUE_KEY_SCD", "ID", "_AIRBYTE_START_AT", "_AIRBYTE_END_AT", "_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID") + +; + commit; \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql new file mode 100644 index 000000000000..198a2627ab68 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql @@ -0,0 +1,25 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_UNIQUE_KEY, + ID, + NAME, + _AB_CDC_LSN, + _AB_CDC_UPDATED_AT, + _AB_CDC_DELETED_AT, + _AB_CDC_LOG_POS, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_POS_DEDUP_CDCX_HASHID +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX_SCD" +-- POS_DEDUP_CDCX from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_POS_DEDUP_CDCX +where 1 = 1 +and _AIRBYTE_ACTIVE_ROW = 1 + ) order by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX" cluster by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql new file mode 100644 index 000000000000..24b58ed5bf4d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql @@ -0,0 +1,99 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX_SCD" as + (select * from( + +with + +input_data as ( + select * + from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB3" + -- POS_DEDUP_CDCX from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_POS_DEDUP_CDCX +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast(ID as + varchar +), '') as + varchar +)) as _AIRBYTE_UNIQUE_KEY, + ID, + NAME, + _AB_CDC_LSN, + _AB_CDC_UPDATED_AT, + _AB_CDC_DELETED_AT, + _AB_CDC_LOG_POS, + _AIRBYTE_EMITTED_AT as _AIRBYTE_START_AT, + lag(_AIRBYTE_EMITTED_AT) over ( + partition by ID + order by + _AIRBYTE_EMITTED_AT is null asc, + _AIRBYTE_EMITTED_AT desc, + _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc, _AB_CDC_LOG_POS desc + ) as _AIRBYTE_END_AT, + case when lag(_AIRBYTE_EMITTED_AT) over ( + partition by ID + order by + _AIRBYTE_EMITTED_AT is null asc, + _AIRBYTE_EMITTED_AT desc, + _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc, _AB_CDC_LOG_POS desc + ) is null and _AB_CDC_DELETED_AT is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + _AIRBYTE_POS_DEDUP_CDCX_HASHID + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT, cast(_AB_CDC_DELETED_AT as + varchar +), cast(_AB_CDC_UPDATED_AT as + varchar +), cast(_AB_CDC_LOG_POS as + varchar +) + order by _AIRBYTE_AB_ID + ) as _AIRBYTE_ROW_NUM, + md5(cast(coalesce(cast(_AIRBYTE_UNIQUE_KEY as + varchar +), '') || '-' || coalesce(cast(_AIRBYTE_START_AT as + varchar +), '') || '-' || coalesce(cast(_AIRBYTE_EMITTED_AT as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_DELETED_AT as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_UPDATED_AT as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_LOG_POS as + varchar +), '') as + varchar +)) as _AIRBYTE_UNIQUE_KEY_SCD, + scd_data.* + from scd_data +) +select + _AIRBYTE_UNIQUE_KEY, + _AIRBYTE_UNIQUE_KEY_SCD, + ID, + NAME, + _AB_CDC_LSN, + _AB_CDC_UPDATED_AT, + _AB_CDC_DELETED_AT, + _AB_CDC_LOG_POS, + _AIRBYTE_START_AT, + _AIRBYTE_END_AT, + _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_POS_DEDUP_CDCX_HASHID +from dedup_data where _AIRBYTE_ROW_NUM = 1 + ) order by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX_SCD" cluster by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql new file mode 100644 index 000000000000..522991af25cd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql @@ -0,0 +1,24 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(NAME as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_LSN as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_UPDATED_AT as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_DELETED_AT as + varchar +), '') as + varchar +)) as _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB2" tmp +-- DEDUP_CDC_EXCLUDED +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql new file mode 100644 index 000000000000..0b361061abf3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql @@ -0,0 +1,30 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(CURRENCY as + varchar +), '') || '-' || coalesce(cast(DATE as + varchar +), '') || '-' || coalesce(cast(TIMESTAMP_COL as + varchar +), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as + varchar +), '') || '-' || coalesce(cast(HKD_SPECIAL___CHARACTERS as + varchar +), '') || '-' || coalesce(cast(NZD as + varchar +), '') || '-' || coalesce(cast(USD as + varchar +), '') as + varchar +)) as _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB2" tmp +-- DEDUP_EXCHANGE_RATE +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql new file mode 100644 index 000000000000..c1189b3c1dfd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql @@ -0,0 +1,25 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(NAME as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_LSN as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_UPDATED_AT as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_DELETED_AT as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_LOG_POS as + varchar +), '') as + varchar +)) as _AIRBYTE_POS_DEDUP_CDCX_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB2" tmp +-- POS_DEDUP_CDCX +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql new file mode 100644 index 000000000000..624b8d286061 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') as + varchar +)) as _AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB2" tmp +-- RENAMED_DEDUP_CDC_EXCLUDED +where 1 = 1 + + );